diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index da3eb3873..7862338f4 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -323,7 +323,7 @@ jobs: persist-credentials: false - name: Download all native candidate pairs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: release-candidates-* path: ${{ runner.temp }}/release-candidate-artifacts diff --git a/.github/workflows/_smoke.yml b/.github/workflows/_smoke.yml index 3e5fa7cc6..c054db4b2 100644 --- a/.github/workflows/_smoke.yml +++ b/.github/workflows/_smoke.yml @@ -111,7 +111,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + # SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper; the + # macOS runner images have none on PATH. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: pkg/go/go.mod + cache: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-${{ matrix.goos }}-${{ matrix.goarch }} @@ -211,7 +218,7 @@ jobs: zip coreutils - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-${{ matrix.arch }} @@ -304,7 +311,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + # SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper. + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: pkg/go/go.mod + cache: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-linux-${{ matrix.arch }}-portable diff --git a/.github/workflows/_soak.yml b/.github/workflows/_soak.yml index 3497fb8f4..870e9f426 100644 --- a/.github/workflows/_soak.yml +++ b/.github/workflows/_soak.yml @@ -72,7 +72,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-${{ matrix.goos }}-${{ matrix.goarch }} path: ${{ runner.temp }}/cbm-selected-container @@ -147,7 +147,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-amd64 path: ${{ runner.temp }}/cbm-selected-container @@ -226,7 +226,7 @@ jobs: run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++ - name: Download selected release container if: ${{ inputs.use_release_artifacts }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-windows-arm64 path: ${{ runner.temp }}/cbm-selected-container @@ -288,7 +288,7 @@ jobs: timeout-minutes: 10 run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y python3 git - name: Download selected portable release container - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: binaries-linux-${{ matrix.arch }}-portable path: ${{ runner.temp }}/cbm-selected-container diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 248f42b47..69e24f340 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -246,7 +246,7 @@ jobs: # producer, so arm64, macOS and Windows code under #ifdef has static # evidence only and those legs run every test. if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: ${{ inputs.test_selection }} path: ${{ runner.temp }}/test-selection @@ -746,7 +746,7 @@ jobs: with: path: repo - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: shard-manifest-* path: manifests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10480dac4..de42a3113 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,7 +231,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download only selected release containers - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: binaries-* merge-multiple: true @@ -240,7 +240,7 @@ jobs: # download above. Preserve only the three reviewable manifests that bind # candidate bytes, VT results, and the tuple-local decision. - name: Download candidate selection evidence separately - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: release-selection-evidence path: selection-evidence diff --git a/Makefile.cbm b/Makefile.cbm index f8274c0bf..49f4079bf 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -419,6 +419,7 @@ EXTRACTION_SRCS = \ $(CBM_DIR)/openapi.c \ $(CBM_DIR)/extract_nextjs.c \ $(CBM_DIR)/sql_values.c \ + $(CBM_DIR)/php_inline_html.c \ $(CBM_DIR)/helpers.c \ $(CBM_DIR)/cpp_branch_views.c \ $(CBM_DIR)/result_compact.c \ diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f87557cc3..4e704ebe0 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -101,6 +101,8 @@ Current keys: | `watcher_enabled` | `true` | Master switch for the retained background watcher subsystem. Set `false` to stop its poll thread/project registration. Reindex manually with `codebase-memory-cli index` when disabled. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | +| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. | +| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). Like `index_max_rss_mb`, it stops a worker only before it starts to publish the new index; see [INDEX_RESOURCE_LIMITS.md](INDEX_RESOURCE_LIMITS.md). | > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the > retained watcher subsystem starts at all. `auto_watch` is narrower: it controls @@ -112,11 +114,13 @@ Current keys: > value. Disabling the watcher does not disable explicit `index`, `search`, > `trace`, `snippet`, or `coverage` commands. -The two `index_max_*` settings are independent and disabled by default. They +The four `index_max_*` settings are independent and disabled by default. They apply to explicit indexing, automatic indexing, and watcher re-indexing, but not to `cross-repo-intelligence`, which does not scan repository source files. -Equality is allowed; exceeding either setting fails the complete index request -and preserves any previously serving database. See +Equality is allowed; exceeding any setting fails the complete index request and +preserves any previously serving database. Worker RSS covers descendants and is +not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total +duration is independent of the existing 15-minute no-log-progress timeout. See [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and error-response details. diff --git a/docs/INDEX_RESOURCE_LIMITS.md b/docs/INDEX_RESOURCE_LIMITS.md index 571eecb76..faa9b0eac 100644 --- a/docs/INDEX_RESOURCE_LIMITS.md +++ b/docs/INDEX_RESOURCE_LIMITS.md @@ -1,8 +1,8 @@ # Index resource limits Index resource limits are optional operator controls for repositories whose -discovery breadth is not known in advance. They are disabled by default so -existing large-repository workloads retain their current behavior. +discovery breadth or worker runtime is not known in advance. They are disabled +by default so existing large-repository workloads retain their current behavior. ## Discovery settings @@ -23,6 +23,61 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero, negative values, suffixes, trailing characters, and values outside the stated ranges are rejected without changing the stored value. +## Worker settings + +| Key | Default | Accepted value | Protects | +|---|---:|---:|---| +| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree | +| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request | + +```bash +codebase-memory-cli config set index_max_rss_mb 8192 +codebase-memory-cli config set index_max_duration_seconds 3600 +``` + +RSS is the charged memory of the contained worker and every descendant, not +the worker's allocation budget and not peak memory. On macOS that quantity is +`phys_footprint` (the same number `cbm_mem_charged()` enforces), not +`resident_size`, which still counts pages the allocator has already handed +back. Linux and Windows use RSS / working set. This hard watchdog is +separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor +samples the tree at most once every 250 milliseconds so the watchdog does not +turn full process-table enumeration into a busy loop. + +Duration is per request: the clock starts at the first worker spawn and is +not reset when crash/hang recovery starts a later attempt. Continuous log +progress does not reset it. It is independent of the existing 15-minute quiet +timeout, which still identifies a worker that stops making progress. A +duration limit shorter than that quiet timeout kills a hung worker before hang +quarantine can name the in-flight file, so the next attempt may hang on the +same file. + +Equality is allowed. The first RSS or elapsed-duration observation above its +limit starts the existing graceful-to-force process-tree shutdown. CBM reports +terminal only after the tree is quiescent or a bounded containment failure is +explicitly surfaced. Resource termination is not retried and does not +quarantine a source file. + +A worker limit never stops a worker after it has started to publish. The +supervisor enforces the limits, but only the worker replaces the serving +database, so the two coordinate through a per-attempt publication claim that +exactly one of them can win. The worker claims it immediately before it +replaces the serving database; the supervisor claims it before it stops the +worker for a limit. If the limit is reached first, the worker can no longer +publish and the previous index stays untouched. If the worker claimed first, +the limits no longer apply to that attempt: the new index is published and the +rest of the attempt, including the persistence artifact export, runs to +completion. The supervisor then logs `index.supervisor.resource_limit_after_publication` +instead of stopping it. With both worker limits `off` there is no claim. + +If RSS is enabled and three consecutive probes cannot obtain any trustworthy +tree measurement while the root worker is still running, CBM fails closed with +`code=resource_probe_failed`. A probe is untrustworthy as soon as one member of +the tree cannot be measured: only a process that exits between the listing and +its measurement is skipped, because a total without a live member would +under-report. On Linux the probe reads every `/proc//stat` to find the +tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed. + ## Counting and failure semantics `index_max_files` counts a file only after it passes directory pruning, ignore @@ -55,6 +110,15 @@ The previous database remains available because publication occurs only after a complete discovery and successful staged build. If no previous database exists, `serving_index_preserved` is false. +Worker limit failures use the same shape with `stage=worker`, +`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and +`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed` +and omit `observed`, `limit`, and `unit` because no trustworthy observation was +available. For a worker failure `serving_index_preserved` describes the whole +request: it is true only when a servable index existed when the request began, +no attempt of the request (including an earlier one that crashed and was +recovered) claimed publication, and that index still serves. + ## Trust and compatibility Limits are read from the CLI-managed `_config.db`; they are not index request @@ -64,5 +128,6 @@ parent policy. These settings do not replace or increase `auto_index_limit`, change the 512 MiB single-file cap, alter workspace-root authorization, or affect -`cross-repo-intelligence`. With both settings `off`, discovery follows the -existing unbounded path. +`cross-repo-intelligence`. With all settings `off`, discovery follows the +existing path and the supervisor performs no periodic RSS probe or total-duration +termination. diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index b5e7c155d..1576431fc 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -3321,6 +3321,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage return r; } +/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup ` (#2000). Same length and line breaks as the input; returns +// `source` itself when nothing needs rewriting. Defined in php_inline_html.c. +const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len); void cbm_extract_imports(CBMExtractCtx *ctx); void cbm_extract_usages(CBMExtractCtx *ctx); void cbm_extract_semantic(CBMExtractCtx *ctx); diff --git a/internal/cbm/extract_calls.c b/internal/cbm/extract_calls.c index 979af928c..1c5b21678 100644 --- a/internal/cbm/extract_calls.c +++ b/internal/cbm/extract_calls.c @@ -4358,6 +4358,70 @@ static TSNode language_specific_callee_leaf(CBMLanguage language, TSNode expr, return terminal_callee_leaf(ctx, expr); } +/* `Base.init` for a `super.init(...)` call: Base is the first entry of the + * enclosing class's inheritance clause, where Swift requires the superclass to + * stand. The class node is carried by the walk, so no ancestor climb. NULL (no + * call) outside a class, without a clause, or when the entry does not read as + * a plain type name -- a lower-case or attributed root would pass the + * registry's receiver-chain check and bind by short name, which is the false + * edge this exists to prevent. */ +static const char *swift_superclass_init_callee(CBMExtractCtx *ctx, const WalkState *state) { + TSNode cls = state ? state->enclosing_class_node : (TSNode){0}; + if (ts_node_is_null(cls) || strcmp(ts_node_type(cls), "class_declaration") != 0) { + return NULL; + } + TSNode spec = cbm_find_child_by_kind(cls, "inheritance_specifier"); + char *base = ts_node_is_null(spec) ? NULL : cbm_node_text(ctx->arena, spec, ctx->source); + if (!base || base[0] < 'A' || base[0] > 'Z') { + return NULL; + } + size_t len = strcspn(base, "<"); /* Base names Base */ + for (size_t i = 0; i < len; i++) { + if (!isalnum((unsigned char)base[i]) && base[i] != '_' && base[i] != '.') { + return NULL; + } + } + base[len] = '\0'; + return cbm_arena_sprintf(ctx->arena, "%s.init", base); +} + +/* True for a Swift initializer callee `T.init` (T non-empty). */ +static bool swift_callee_is_initializer(const char *callee) { + static const char suffix[] = ".init"; + size_t n = callee ? strlen(callee) : 0; + return n > sizeof(suffix) - 1 && strcmp(callee + n - (sizeof(suffix) - 1), suffix) == 0; +} + +/* Swift initializer delegation (#2379). `super.init(...)` and `self.init(...)` + * name no type, and a lower-case receiver passes the registry's receiver-chain + * check, so a short-name match binds them to whichever project `init` is + * nearest: a nested type's, a sibling type's, or the caller's own. On + * Alamofire all 8 such edges were false. + * + * `super.init` is rewritten to name the superclass (`Base.init`), so the + * qualified strategies place it on the project's Base.init. When Base is not + * in the project (UIViewController, NSObject) nothing places it, and the + * initializer guard in handle_calls keeps a short-name guess from doing so. + * + * `self.init` delegates to another initializer of the caller's own type. + * Initializer overloads share one node (#2061), so the only edge it could form + * is a self-loop, and that one is false whenever the overload it calls lives + * outside the project (`extension Notification { init(...) { self.init(...) } }` + * calls Foundation's). It forms no CALLS edge. */ +static const char *resolve_swift_delegating_init(CBMExtractCtx *ctx, const WalkState *state, + const char *callee) { + if (ctx->language != CBM_LANG_SWIFT || !callee) { + return callee; + } + if (strcmp(callee, "self.init") == 0) { + return NULL; + } + if (strcmp(callee, "super.init") == 0) { + return swift_superclass_init_callee(ctx, state); + } + return callee; +} + // Produce the textual callee and the AST occurrence from one operation. The // generic fallback retains the former field/terminal behavior, while grammars // whose extractor uses a sibling, a special field, or a synthetic literal are @@ -4368,6 +4432,7 @@ static CBMPrimaryCalleeSelection select_primary_callee(CBMExtractCtx *ctx, TSNod CBMPrimaryCalleeSelection selection = {0}; selection.name = extract_callee_name(ctx->arena, node, ctx->source, ctx->language); selection.name = resolve_objectscript_callee(ctx, node, state, selection.name); + selection.name = resolve_swift_delegating_init(ctx, state, selection.name); if (!selection.name || !selection.name[0]) { return selection; } @@ -4591,6 +4656,16 @@ CBMInvocationDescriptor handle_calls(CBMExtractCtx *ctx, TSNode node, const CBML call.is_method = true; } } + // Swift initializer guard (#2379; same direction). Flag `T.init(...)`, + // as written or as a rewritten super.init. Nearly every type has an + // `init`, so a short-name match onto one is a guess: with a project + // module imported, `UIViewController.init` bound an unrelated + // project initializer. The registry's qualified / same-module / + // import strategies still place T.init when T is a project type. + // No other Swift call carries the flag. + if (ctx->language == CBM_LANG_SWIFT && swift_callee_is_initializer(call.callee_name)) { + call.is_method = true; + } TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); /* tree-sitter-elixir attaches NO field name to a call's arguments diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index c0d58adb9..ef778c260 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -800,6 +800,15 @@ TSNode cbm_resolve_func_name(TSNode node, CBMLanguage lang) { return cbm_find_child_by_kind(node, "query_name"); } + /* Swift init/deinit (#2379) have no name node at all. The declaration + * itself stands in for it, and cbm_func_name_node_text maps that + * node kind to "init" / "deinit". Gated on Swift: Slang's grammar + * also has an init_declaration node. */ + if (lang == CBM_LANG_SWIFT && + (strcmp(kind, "init_declaration") == 0 || strcmp(kind, "deinit_declaration") == 0)) { + return node; + } + TSNode name = func_name_node(node); if (lang == CBM_LANG_R && strcmp(kind, "function_definition") == 0) { @@ -9210,6 +9219,12 @@ static TSNode resolve_method_name(TSNode child, CBMLanguage lang) { return cbm_find_child_by_kind(child, "simple_identifier"); } + // Swift init/deinit: named in one place, cbm_resolve_func_name (#2379). + if (lang == CBM_LANG_SWIFT && + (strcmp(ck, "init_declaration") == 0 || strcmp(ck, "deinit_declaration") == 0)) { + return cbm_resolve_func_name(child, lang); + } + // Squirrel: function_declaration's name is a plain `identifier` child. if (lang == CBM_LANG_SQUIRREL && strcmp(ck, "function_declaration") == 0) { return cbm_find_child_by_kind(child, "identifier"); diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index 857a43ea1..6c27719d3 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -324,6 +324,7 @@ static bool push_scope(WalkState *state, uint8_t kind, uint32_t depth, const cha * CBMWalkScope comment for the quadratic recompute this replaces. */ f->prev_enclosing_func_qn = state->enclosing_func_qn; f->prev_enclosing_class_qn = state->enclosing_class_qn; + f->prev_enclosing_class_node = state->enclosing_class_node; f->prev_invocation_kind = state->invocation_kind; f->prev_callee_expr = state->callee_expr; f->prev_callee_leaf = state->callee_leaf; @@ -337,6 +338,8 @@ static bool push_scope(WalkState *state, uint8_t kind, uint32_t depth, const cha case SCOPE_CLASS: case SCOPE_NAMESPACE: state->enclosing_class_qn = qn; + /* push_lexical_scope records the node; a node-less push has none. */ + state->enclosing_class_node = (TSNode){0}; break; case SCOPE_CALL: /* The invocation triple is applied by push_call_scope once the caller @@ -364,6 +367,9 @@ static bool push_lexical_scope(WalkState *state, uint8_t walk_kind, uint32_t dep state->lexical_binding_tracking_failed = true; return false; } + if (walk_kind == SCOPE_CLASS || walk_kind == SCOPE_NAMESPACE) { + state->enclosing_class_node = node; + } state->scopes[state->scope_top - SKIP_ONE].lexical_scope_id = add_lexical_scope(state, node, lexical_kind); return true; @@ -442,6 +448,7 @@ static void pop_expired_scopes(WalkState *state, uint32_t cur_depth) { const CBMWalkScope *f = &state->scopes[--state->scope_top]; state->enclosing_func_qn = f->prev_enclosing_func_qn; state->enclosing_class_qn = f->prev_enclosing_class_qn; + state->enclosing_class_node = f->prev_enclosing_class_node; state->invocation_kind = f->prev_invocation_kind; state->callee_expr = f->prev_callee_expr; state->callee_leaf = f->prev_callee_leaf; @@ -2823,6 +2830,7 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { * recompute_state call): module scope, nothing else active. */ state.enclosing_func_qn = ctx->module_qn; state.enclosing_class_qn = NULL; + state.enclosing_class_node = (TSNode){0}; state.invocation_kind = CBM_INVOCATION_NONE; state.callee_expr = (TSNode){0}; state.callee_leaf = (TSNode){0}; diff --git a/internal/cbm/extract_unified.h b/internal/cbm/extract_unified.h index f80237229..8e40d6a2c 100644 --- a/internal/cbm/extract_unified.h +++ b/internal/cbm/extract_unified.h @@ -78,6 +78,7 @@ typedef struct { * suite-budget kills on every non-M4 venue). */ const char *prev_enclosing_func_qn; const char *prev_enclosing_class_qn; + TSNode prev_enclosing_class_node; CBMInvocationKind prev_invocation_kind; TSNode prev_callee_expr; TSNode prev_callee_leaf; @@ -133,6 +134,7 @@ typedef struct { typedef struct { const char *enclosing_func_qn; // current function QN (module_qn at top level) const char *enclosing_class_qn; // current class QN (NULL outside class) + TSNode enclosing_class_node; // that class scope's node (null when not known) const TSTreeCursor *current_cursor; // unified walk cursor at the current node TSTreeCursor *occurrence_cursor; // reusable parent-preserving classifier cursor CBMInvocationKind invocation_kind; // exact active invocation/reference role diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index 20a876106..b6cd48936 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -301,6 +301,16 @@ bool cbm_is_keyword(const char *name, CBMLanguage lang) { case CBM_LANG_PUPPET: keywords = puppet_keywords; break; + case CBM_LANG_SWIFT: + /* `init` and `deinit` are Swift keywords. A reference to one + * (`.init(...)`, `Foo.init`) names an initializer of a type the bare + * name does not carry, so resolving it by name can only guess among + * every type's initializer (#2379). Otherwise the generic set. */ + if (strcmp(name, "init") == 0 || strcmp(name, "deinit") == 0) { + return true; + } + keywords = generic_keywords; + break; default: keywords = generic_keywords; break; @@ -1198,6 +1208,16 @@ bool cbm_is_c_preprocessor_lang(CBMLanguage lang) { // const", and a member lookup for "operator bool" (the implicit call in // `if (obj)`) misses. char *cbm_func_name_node_text(CBMArena *a, TSNode name_node, const char *source, CBMLanguage lang) { + if (lang == CBM_LANG_SWIFT) { + const char *kind = ts_node_type(name_node); + if (strcmp(kind, "init_declaration") == 0) { + return cbm_arena_strdup(a, "init"); + } + if (strcmp(kind, "deinit_declaration") == 0) { + return cbm_arena_strdup(a, "deinit"); + } + } + char *text = cbm_node_text(a, name_node, source); /* A C keyword is never a function name: the definition is an error-recovery * artifact. No name means no def and no call scope (calls inside fall back diff --git a/internal/cbm/lang_specs.c b/internal/cbm/lang_specs.c index 053c21884..150a4e39a 100644 --- a/internal/cbm/lang_specs.c +++ b/internal/cbm/lang_specs.c @@ -571,7 +571,8 @@ static const char *objc_assign_types[] = {"assignment_expression", NULL}; // well as in the name resolvers — extract_class_methods gates on this set, so // without the entry the requirement is walked and then discarded. static const char *swift_func_types[] = {"function_declaration", "protocol_function_declaration", - "macro_declaration", NULL}; + "macro_declaration", "init_declaration", + "deinit_declaration", NULL}; // KNOWN GAP: struct_declaration and enum_declaration are not node types in the // vendored tree-sitter-swift grammar — it models both as class_declaration — so // these two entries never match anything, and a bare Swift `enum` is labeled diff --git a/internal/cbm/lsp/php_lsp.c b/internal/cbm/lsp/php_lsp.c index 7aae329e8..85118f671 100644 --- a/internal/cbm/lsp/php_lsp.c +++ b/internal/cbm/lsp/php_lsp.c @@ -20,6 +20,7 @@ #include "php_lsp.h" #include "lsp_node_iter.h" #include "lsp_work.h" +#include "../cbm.h" #include "../helpers.h" #include "../../../src/foundation/hash_table.h" #include @@ -4795,6 +4796,9 @@ void cbm_run_php_lsp_cross_with_bindings(CBMArena *arena, const char *source, in const CBMPHPImportBindings *bindings) { if (!arena || !source || source_len <= 0 || !out) return; + /* The retained source is the file on disk; read it the way the per-file + * extract did, so the cached tree and node text agree (#2000). */ + source = cbm_php_mask_inline_html(arena, source, source_len); TSParser *parser = NULL; TSTree *tree = cached_tree; diff --git a/internal/cbm/php_inline_html.c b/internal/cbm/php_inline_html.c new file mode 100644 index 000000000..6c1374fc4 --- /dev/null +++ b/internal/cbm/php_inline_html.c @@ -0,0 +1,371 @@ +// php_inline_html.c — let the php_only grammar read a file that leaves PHP mode. +// +// A PHP file starts in inline-HTML mode and may switch in and out of PHP any +// number of times: `?>
...
` or for the +// text that follows it, so every byte after the first close tag used to land in +// an ERROR region and the declarations there never reached the graph (#2000). +// +// Instead of parsing the markup, blank it. The rewritten buffer has the same +// length as the original and keeps every line break, so byte offsets, rows and +// columns are unchanged and node text inside PHP code is byte-identical: +// +// - markup before the first open tag and between tags becomes spaces +// - the first open tag stays, so the tree keeps its php_tag +// - each later `` becomes `; `, the statement terminator PHP itself treats it as +// +// Only code and `//` / `#` comments can close PHP mode. Strings, heredoc, +// nowdoc and block comments are skipped, `#[` is an attribute, and everything +// after __halt_compiler is left alone, as PHP stops reading there. + +#include "cbm.h" +#include "foundation/arena.h" +#include "foundation/constants.h" +#include +#include +#include + +enum { + PHP_TAG_OPEN_LEN = 5, /* = 0 && i <= len && (size_t)(len - i) >= n && memcmp(s + i, lit, n) == 0; +} + +static bool php_is_ident_char(unsigned char c) { + return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c >= PHP_FIRST_NON_ASCII; +} + +static bool php_is_ident_start(unsigned char c) { + return php_is_ident_char(c) && !(c >= '0' && c <= '9'); +} + +static bool php_is_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; +} + +static bool php_is_blank(char c) { + return c == ' ' || c == '\t'; +} + +/* Case-insensitive match of the lowercase `word` at s[i]. */ +static bool php_ieq(const char *s, int len, int i, const char *word) { + int n = (int)strlen(word); + if (i + n > len) { + return false; + } + for (int k = 0; k < n; k++) { + char c = s[i + k]; + if (c >= 'A' && c <= 'Z') { + c = (char)(c - 'A' + 'a'); + } + if (c != word[k]) { + return false; + } + } + return true; +} + +/* True when the identifier at i is `xml` (an XML prolog target). */ +static bool php_xml_prolog(const char *s, int len, int i) { + return php_ieq(s, len, i, "xml") && + !(i + PHP_XML_LEN < len && php_is_ident_char((unsigned char)s[i + PHP_XML_LEN])); +} + +/* Length of the open tag at i, or 0. `")) { + return true; + } + } + return false; +} + +/* Blank [from, to) in place, keeping line breaks. */ +static void php_blank(char *out, int from, int to) { + for (int k = from; k < to; k++) { + if (out[k] != '\n' && out[k] != '\r') { + out[k] = ' '; + } + } +} + +/* Index just past the quoted string opened at i. */ +static int php_skip_quoted(const char *s, int len, int i) { + char quote = s[i]; + i += SKIP_ONE; + while (i < len && s[i] != quote) { + i += (s[i] == '\\') ? PAIR_LEN : SKIP_ONE; + } + return i < len ? i + SKIP_ONE : len; +} + +/* Index just past the block comment opened at i. */ +static int php_skip_block_comment(const char *s, int len, int i) { + for (int k = i + PAIR_LEN; k < len; k++) { + if (php_at(s, len, k, "*/")) { + return k + PAIR_LEN; + } + } + return len; +} + +/* Index of the line break or `?>` that ends the one-line comment at i. */ +static int php_skip_line_comment(const char *s, int len, int i) { + while (i < len && s[i] != '\n' && s[i] != '\r' && !php_at(s, len, i, "?>")) { + i++; + } + return i; +} + +/* Parse the heredoc/nowdoc opener `<<= len || s[p] != quote) { + return PHP_NOT_FOUND; + } + p++; + } + if (p < len && s[p] == '\r') { + p++; + } + return (p < len && s[p] == '\n') ? p : PHP_NOT_FOUND; +} + +/* True when the line starting at p closes the heredoc. PHP 7.3+ allows the + * closing label to be indented and followed by any non-identifier byte. */ +static bool php_heredoc_closes(const char *s, int len, int p, int label, int label_len, int *end) { + while (p < len && php_is_blank(s[p])) { + p++; + } + if (len - p < label_len || memcmp(s + p, s + label, (size_t)label_len) != 0) { + return false; + } + int after = p + label_len; + if (after < len && php_is_ident_char((unsigned char)s[after])) { + return false; + } + *end = after; + return true; +} + +/* Index just past the heredoc/nowdoc opened by the `<<<` at i. */ +static int php_skip_heredoc(const char *s, int len, int i) { + int label = 0; + int label_len = 0; + int nl = php_heredoc_opener(s, len, i, &label, &label_len); + if (nl < 0) { + return i + PHP_HEREDOC_LEN; + } + while (nl < len) { + int line = nl + SKIP_ONE; + int end = 0; + if (php_heredoc_closes(s, len, line, label, label_len, &end)) { + return end; + } + const char *next = memchr(s + line, '\n', (size_t)(len - line)); + if (!next) { + return len; + } + nl = (int)(next - s); + } + return len; +} + +/* Index just past the identifier at i; sets *halt when it is __halt_compiler + * (a keyword, so not a `$variable` of that name). */ +static int php_skip_ident(const char *s, int len, int i, bool *halt) { + int start = i; + while (i < len && php_is_ident_char((unsigned char)s[i])) { + i++; + } + bool is_var = start > 0 && s[start - SKIP_ONE] == '$'; + *halt = + !is_var && i - start == PHP_HALT_COMPILER_LEN && php_ieq(s, len, start, PHP_HALT_COMPILER); + return i; +} + +/* Scan PHP code from i to the next close tag. Returns the index of that `?>`, + * len when the file ends in PHP mode, or PHP_NOT_FOUND at __halt_compiler. */ +static int php_scan_code(const char *s, int len, int i, int *comment) { + *comment = PHP_NOT_FOUND; + while (i < len) { + unsigned char c = (unsigned char)s[i]; + if (php_at(s, len, i, "?>")) { + return i; + } + if (c == '\'' || c == '"' || c == '`') { + i = php_skip_quoted(s, len, i); + } else if (php_at(s, len, i, "/*")) { + i = php_skip_block_comment(s, len, i); + } else if (php_at(s, len, i, "//") || (c == '#' && !php_at(s, len, i, "#["))) { + int start = i; + i = php_skip_line_comment(s, len, i); + if (php_at(s, len, i, "?>")) { + *comment = start; /* the close tag ends this comment */ + } + } else if (php_at(s, len, i, "<<<")) { + i = php_skip_heredoc(s, len, i); + } else if (php_is_ident_start(c)) { + bool halt = false; + i = php_skip_ident(s, len, i, &halt); + if (halt) { + return PHP_NOT_FOUND; + } + } else { + i++; + } + } + return len; +} + +/* The rewritten buffer. It is allocated and filled from the source on the + * first write, so a file whose only `?>` bytes sit in strings or comments + * costs no copy. */ +typedef struct { + CBMArena *arena; + const char *source; + int len; + char *out; + bool failed; +} PHPMask; + +static char *php_mask_writable(PHPMask *m) { + if (!m->out && !m->failed) { + m->out = (char *)cbm_arena_alloc(m->arena, (size_t)m->len + SKIP_ONE); + if (m->out) { + memcpy(m->out, m->source, (size_t)m->len); + m->out[m->len] = '\0'; + } else { + m->failed = true; + } + } + return m->out; +} + +/* Blank [from, to) of the rewritten buffer, keeping line breaks. */ +static void php_mask_blank(PHPMask *m, int from, int to) { + char *out = php_mask_writable(m); + if (out) { + php_blank(out, from, to); + } +} + +/* Turn the `?>` at i into `; `. When the tag ends a line comment starting at + * `comment`, blank that comment first: `; ` cannot end it and a line break + * would shift rows, so the reopened PHP on the same line would be swallowed. */ +static void php_mask_close_tag(PHPMask *m, int i, int comment) { + char *out = php_mask_writable(m); + if (out) { + if (comment >= 0) { + php_blank(out, comment, i); + } + out[i] = ';'; + out[i + SKIP_ONE] = ' '; + } +} + +const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len) { + if (!arena || !source || source_len <= 0) { + return source; + } + /* Fast path: a file that opens PHP at its first byte and never writes a + * close tag is exactly what the grammar already accepts. */ + if (php_open_tag_len(source, source_len, 0) > 0 && !php_has_close_tag(source, source_len)) { + return source; + } + + PHPMask m = {.arena = arena, .source = source, .len = source_len}; + bool seen_open = false; + int i = 0; + while (i < source_len && !m.failed) { + /* Inline-HTML mode: blank up to the next open tag. */ + int html_start = i; + int tag = 0; + while (i < source_len && (tag = php_open_tag_len(source, source_len, i)) == 0) { + i++; + } + if (i > html_start) { + php_mask_blank(&m, html_start, i); + } + if (i >= source_len) { + break; + } + if (seen_open) { + php_mask_blank(&m, i, i + tag); + } + seen_open = true; + + /* PHP mode, up to the close tag. */ + int comment = PHP_NOT_FOUND; + int close = php_scan_code(source, source_len, i + tag, &comment); + if (close < 0 || close >= source_len) { + break; + } + php_mask_close_tag(&m, close, comment); + i = close + PAIR_LEN; + } + /* On allocation failure parse the file as it is: the old behaviour. */ + return (m.out && !m.failed) ? m.out : source; +} diff --git a/scripts/lint-mem-gate.py b/scripts/lint-mem-gate.py index 140062f37..d5ccd76d2 100755 --- a/scripts/lint-mem-gate.py +++ b/scripts/lint-mem-gate.py @@ -47,7 +47,15 @@ MIN_WHY = 120 MIN_TRIED = 40 -FINDING_RE = re.compile(r"^(?P[^:]+):(?P\d+):\d+:\s+(?:error|warning):\s+(?P.*?)\s*\[(?P[\w.-]+)\]\s*$") +# With WarningsAsErrors enabled, clang-tidy appends the escalation reason to +# analyzer checks, for example: +# [clang-analyzer-core.NullDereference,-warnings-as-errors] +# Keep the check name separate from that suffix so the gate still matches the +# same whitelist key instead of silently dropping the finding. +FINDING_RE = re.compile( + r"^(?P[^:]+):(?P\d+):\d+:\s+(?:error|warning):\s+" + r"(?P.*?)\s*\[(?P[\w.-]+)(?:,-warnings-as-errors)?\]\s*$" +) def strip_noise(text): diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index c76f7d93e..638947ef8 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -58,7 +58,7 @@ src/operations/compare.c 3 src/operations/cross_repo.c 22 src/operations/execute.c 2 src/operations/file_outline.c 1 -src/operations/index.c 56 +src/operations/index.c 46 src/operations/index_supervisor.c 8 src/operations/project_arg.c 2 src/operations/query.c 4 diff --git a/scripts/test-runtime.sh b/scripts/test-runtime.sh index 5d2c0c3ae..e6e60b15a 100644 --- a/scripts/test-runtime.sh +++ b/scripts/test-runtime.sh @@ -112,7 +112,8 @@ _cbm_test_runtime_daemon() { # cbm_mcp_index_policy_add_to_args: every key in cbm_index_policy_key_at, and # nothing else. cbm_test_index_worker_policy_json() { - printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' + printf '%s' '"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"' + printf '%s' ',"index_max_rss_mb":"off","index_max_duration_seconds":"off"}' } cbm_test_runtime_cleanup() { diff --git a/scripts/test.sh b/scripts/test.sh index b7f25dc8c..9d3d182d1 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -354,6 +354,9 @@ bash "$ROOT/tests/test_activation_diagnostic_contract.sh" echo "=== Step 0e3: security gate fail-closed contract ===" bash "$ROOT/tests/test_security_gate_fail_closed.sh" +echo "=== Step 0e4: memory-analyzer gate warning-suffix contract (#2442) ===" +bash "$ROOT/tests/test_lint_mem_gate.sh" + echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" diff --git a/src/cli/cli.c b/src/cli/cli.c index e0a85245d..5172beff4 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7513,6 +7513,9 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, {CBM_INDEX_CONFIG_MAX_FILES, "off", "Max accepted source files per index, or off"}, {CBM_INDEX_CONFIG_MAX_SOURCE_MB, "off", "Max accepted source MiB per index, or off"}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, "off", "Max worker process-tree charged memory MiB, or off"}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "off", + "Max wall-clock seconds for the whole index request, or off"}, }; /* #1558: ui_enabled and ui_port were reachable ONLY by hand-editing @@ -7539,8 +7542,12 @@ static bool config_key_is_ui(const char *key) { } static bool config_key_is_index_policy(const char *key) { - return key && (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0 || - strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + for (size_t index = 0; key && index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, cbm_index_policy_key_at(index)) == 0) { + return true; + } + } + return false; } static int config_index_policy_write(cbm_config_t *config, const char *key, const char *value) { diff --git a/src/cli/config_yaml_edit.c b/src/cli/config_yaml_edit.c index a6f8ced10..09a079212 100644 --- a/src/cli/config_yaml_edit.c +++ b/src/cli/config_yaml_edit.c @@ -2589,6 +2589,10 @@ typedef struct { bool identity_found; size_t identity_start; size_t identity_end; + /* The missing rest of the path begins under a `parent: {}` header. */ + bool parent_inline_empty; + size_t parent_line; + size_t parent_colon; } yaml_mapping_sequence_target_t; static void yaml_sequence_key_vec_free(yaml_sequence_key_vec_t *keys) { @@ -2962,6 +2966,37 @@ static int yaml_sequence_parse_block(const yaml_doc_t *doc, size_t begin_line, s return 0; } +/* `section: {}` is how the mapping-entry editor writes a section it emptied + * (yaml_build_last_mapping_entry_removal), and its upsert accepts it back. On + * a PARENT of the sequence path it is that same empty mapping, so the rest of + * the path is missing beneath it: removal is a no-op and upsert expands the + * header in place. The Hermes remover leaves `hooks: {}` whenever our hook was + * the only entry. The sequence key itself holding `{}`, any other flow value, + * and content nested under the inline mapping stay refused. */ +static int yaml_sequence_target_empty_parent(const yaml_doc_t *doc, size_t key_line, size_t colon, + size_t child_end, size_t next_depth, + size_t sequence_path_len, + yaml_mapping_sequence_target_t *target) { + bool inline_empty = false; + if (next_depth >= sequence_path_len || + yaml_tail_is_explicit_empty_mapping(doc->data, colon, doc->lines[key_line].text_end, + &inline_empty) != 0 || + !inline_empty) { + return YAML_ERROR; + } + for (size_t i = key_line + YAML_UNIT; i < child_end; i++) { + if (!doc->lines[i].blank && !doc->lines[i].comment) { + return YAML_ERROR; + } + } + target->missing_index = next_depth; + target->insert_offset = yaml_sequence_line_offset(doc, child_end); + target->parent_inline_empty = true; + target->parent_line = key_line; + target->parent_colon = colon; + return 0; +} + static int yaml_sequence_analyze(const yaml_doc_t *doc, const char *const *sequence_path, const size_t *path_lengths, size_t sequence_path_len, const char *identity_key, const char *identity_value, @@ -2993,10 +3028,11 @@ static int yaml_sequence_analyze(const yaml_doc_t *doc, const char *const *seque return 0; } const yaml_line_t *header = &doc->lines[key_line]; + size_t child_end = yaml_sequence_nested_end(doc, key_line, parent_end); if (!yaml_tail_is_empty(doc->data, colon, header->text_end)) { - return YAML_ERROR; + return yaml_sequence_target_empty_parent(doc, key_line, colon, child_end, + depth + YAML_UNIT, sequence_path_len, target); } - size_t child_end = yaml_sequence_nested_end(doc, key_line, parent_end); if (depth + YAML_UNIT == sequence_path_len) { target->sequence_found = true; target->missing_index = sequence_path_len; @@ -3095,13 +3131,52 @@ static int yaml_sequence_validate_canonical(const char *canonical_item, size_t c return result; } +/* Copies the document up to insert_offset with the `parent: {}` header turned + * into a block header `parent:`, exactly as + * yaml_build_inline_empty_mapping_section_add does for a mapping section: a + * trailing comment and the line ending are kept. */ +static int yaml_sequence_append_expanded_parent(yaml_buf_t *out, const yaml_doc_t *doc, + const yaml_mapping_sequence_target_t *target) { + if (target->parent_line >= doc->line_count) { + return YAML_ERROR; + } + const yaml_line_t *header = &doc->lines[target->parent_line]; + size_t comment = 0U; + if (target->insert_offset < header->end || + yaml_find_comment(doc->data, target->parent_colon + YAML_UNIT, header->text_end, + &comment) != 0 || + yaml_buf_append(out, doc->data, target->parent_colon + YAML_UNIT) != 0) { + return YAML_ERROR; + } + size_t kept_tail = header->text_end; + if (comment < header->text_end) { + if (yaml_buf_append_char(out, ' ') != 0) { + return YAML_ERROR; + } + kept_tail = comment; + } + if (yaml_buf_append(out, doc->data + kept_tail, header->end - kept_tail) != 0 || + yaml_buf_append(out, doc->data + header->end, target->insert_offset - header->end) != 0 || + (out->len > 0U && out->data[out->len - YAML_UNIT] != '\n' && + yaml_buf_append(out, doc->eol, doc->eol_len) != 0)) { + return YAML_ERROR; + } + return 0; +} + static int yaml_sequence_build_insert(yaml_buf_t *out, const yaml_doc_t *doc, const yaml_mapping_sequence_target_t *target, const char *const *sequence_path, const size_t *path_lengths, size_t sequence_path_len, const yaml_buf_t *rendered_item) { - if (target->insert_offset > doc->len || - yaml_buf_append(out, doc->data, target->insert_offset) != 0 || - yaml_append_separator_if_needed(out, doc, target->insert_offset) != 0) { + if (target->insert_offset > doc->len) { + return YAML_ERROR; + } + if (target->parent_inline_empty) { + if (yaml_sequence_append_expanded_parent(out, doc, target) != 0) { + return YAML_ERROR; + } + } else if (yaml_buf_append(out, doc->data, target->insert_offset) != 0 || + yaml_append_separator_if_needed(out, doc, target->insert_offset) != 0) { return YAML_ERROR; } if (!target->sequence_found) { diff --git a/src/daemon/application.c b/src/daemon/application.c index 46f1c47ea..32b1b51fb 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -162,6 +162,11 @@ struct cbm_daemon_application_job { bool async_owned; bool worker_started; application_index_watch_owner_t *watch_owners; + uint64_t request_started_ms; + /* What a worker resource failure of this job is reported against; set + * before the first attempt, folded over every attempt (operations/index.h). */ + bool request_facts_begun; + cbm_index_request_facts_t request_facts; cbm_daemon_application_job_t *next; }; @@ -506,8 +511,16 @@ static void application_worker_diagnostic(const char *line, void *context) { static int application_worker_start_default(void *context, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { (void)context; + cbm_index_resource_policy_t resource_policy; + char error[CBM_SZ_256] = {0}; + if (!cbm_index_policy_from_internal_args(args_json, &resource_policy, error, sizeof(error))) { + cbm_log_error("daemon.index.policy", "error", error); + *worker_out = NULL; + return -1; + } application_default_worker_t *worker = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*worker)); if (!worker) { return -1; @@ -517,9 +530,9 @@ static int application_worker_start_default(void *context, const char *args_json const char *root = yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args_doc), "repo_path")); worker->project = application_index_project_key(root, args_json); yyjson_doc_free(args_doc); - int result = cbm_index_worker_start_with_log(args_json, memory_budget_bytes, false, marker_file, - quarantine_file, application_worker_diagnostic, - worker, &worker->handle); + int result = cbm_index_worker_start_with_policy_log( + args_json, memory_budget_bytes, &resource_policy, false, marker_file, quarantine_file, + application_worker_diagnostic, worker, duration_origin_ms, &worker->handle); if (result != 0) { application_worker_destroy_default(NULL, worker); *worker_out = NULL; @@ -1671,6 +1684,17 @@ static void application_cancelled_result(cbm_index_worker_result_t *result) { result->tree_quiesced = true; } +/* Only an enabled worker limit can end a job in a worker resource failure, the + * one response the request facts feed; without one nothing is read. */ +static void application_job_request_facts_begin(cbm_daemon_application_job_t *job) { + cbm_index_resource_policy_t policy; + char error[CBM_SZ_256] = {0}; + if (cbm_index_policy_from_internal_args(job->args_json, &policy, error, sizeof(error)) && + cbm_index_policy_worker_enabled(&policy)) { + cbm_index_request_facts_begin(job->args_json, &job->request_facts); + } +} + static application_attempt_status_t application_job_run_attempt(cbm_daemon_application_job_t *job, const char *marker_path, const char *quarantine_path, @@ -1704,10 +1728,17 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli } cbm_daemon_application_worker_t worker = NULL; + if (job->request_started_ms == 0) { + job->request_started_ms = cbm_index_worker_now_ms(); + } + if (!job->request_facts_begun) { + job->request_facts_begun = true; + application_job_request_facts_begin(job); + } application_tmp_lock(); - int start_result = - application->worker_ops.start(application->worker_ops.context, job->args_json, - memory_budget_bytes, marker_path, quarantine_path, &worker); + int start_result = application->worker_ops.start( + application->worker_ops.context, job->args_json, memory_budget_bytes, marker_path, + quarantine_path, job->request_started_ms, &worker); application_tmp_unlock(); if (start_result != 0 || !worker) { return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED @@ -1992,6 +2023,8 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_disposition(job, attempt); application_record_attempt(attempt, &execution->last_result, &execution->have_last_result, execution->last_log); + cbm_index_request_facts_record(&job->request_facts, + attempt->has_result ? &attempt->result : NULL); if (disposition == CBM_INDEX_SUPERVISED_RESULT_SUCCESS) { execution->response = attempt->result.response; attempt->result.response = NULL; @@ -1999,6 +2032,12 @@ static application_attempt_decision_t application_consume_attempt( application_attempt_free(attempt); return APPLICATION_ATTEMPT_DECISION_SUCCESS; } + if (disposition == CBM_INDEX_SUPERVISED_RESULT_RESOURCE_FAILURE) { + execution->response = cbm_index_worker_resource_response(job->args_json, &attempt->result, + &job->request_facts); + application_attempt_free(attempt); + return APPLICATION_ATTEMPT_DECISION_STOP; + } if (disposition == CBM_INDEX_SUPERVISED_RESULT_UNSAFE_TERMINAL) { execution->unsafe_terminal = true; execution->supervision_failed = @@ -3926,7 +3965,7 @@ cbm_daemon_runtime_application_callbacks_t cbm_daemon_application_runtime_callba static cbm_daemon_runtime_application_status_t application_client_exchange_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, uint8_t *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { uint8_t *response = NULL; uint32_t response_length = 0; if (response_out) { @@ -3935,13 +3974,16 @@ static cbm_daemon_runtime_application_status_t application_client_exchange_tagge if (response_length_out) { *response_length_out = 0; } + if (request_sent_out) { + *request_sent_out = false; + } cbm_daemon_runtime_application_status_t status = request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID ? cbm_daemon_runtime_client_application_request(client, request, request_length, &response, &response_length, timeout_ms) - : cbm_daemon_runtime_client_application_request_tagged(client, request_token, request, - request_length, &response, - &response_length, timeout_ms); + : cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, request_length, &response, &response_length, + request_sent_out, timeout_ms); free(request); if (status != CBM_DAEMON_RUNTIME_APPLICATION_OK) { free(response); @@ -3972,7 +4014,7 @@ static cbm_daemon_runtime_application_status_t application_client_exchange( uint8_t **response_out, uint32_t *response_length_out, uint32_t timeout_ms) { return application_client_exchange_tagged(client, CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID, request, request_length, response_out, - response_length_out, timeout_ms); + response_length_out, NULL, timeout_ms); } cbm_daemon_runtime_application_status_t cbm_daemon_application_client_set_context( @@ -4096,7 +4138,7 @@ cbm_daemon_runtime_application_status_t cbm_daemon_application_client_ui_readine static cbm_daemon_runtime_application_status_t application_client_text_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, cbm_daemon_application_request_kind_t kind, const char *text, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { if (!client || !text || !text[0]) { return CBM_DAEMON_RUNTIME_APPLICATION_REJECTED; } @@ -4112,7 +4154,7 @@ static cbm_daemon_runtime_application_status_t application_client_text_request_t memcpy(request + 1, text, text_length); return application_client_exchange_tagged(client, request_token, request, (uint32_t)text_length + 1U, response_out, - response_length_out, timeout_ms); + response_length_out, request_sent_out, timeout_ms); } static cbm_daemon_runtime_application_status_t application_client_text_request( @@ -4120,7 +4162,7 @@ static cbm_daemon_runtime_application_status_t application_client_text_request( const char *text, uint8_t **response_out, uint32_t *response_length_out, uint32_t timeout_ms) { return application_client_text_request_tagged( client, CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID, kind, text, response_out, - response_length_out, timeout_ms); + response_length_out, NULL, timeout_ms); } cbm_daemon_runtime_application_status_t cbm_daemon_application_client_operation_result( diff --git a/src/daemon/application.h b/src/daemon/application.h index a0bc2a691..17b5c52a9 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -28,7 +28,7 @@ typedef void *cbm_daemon_application_worker_t; typedef struct { void *context; int (*start)(void *context, const char *args_json, size_t memory_budget_bytes, - const char *marker_file, const char *quarantine_file, + const char *marker_file, const char *quarantine_file, uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out); cbm_index_worker_poll_t (*poll)(void *context, cbm_daemon_application_worker_t worker, const cbm_index_worker_result_t **result_out); diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index 9035e7f5d..df0eb8cbf 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -141,6 +141,12 @@ static bool bootstrap_worker_argv_exact(int argc, char *const argv[]) { } next += 2; } + if (next < argc && bootstrap_arg_is(argv[next], "--index-worker-publish-fence")) { + if (next + 1 >= argc || !argv[next + 1] || !argv[next + 1][0]) { + return false; + } + next += 2; + } return next == argc; } diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 9ee80df87..3a1488961 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -3403,13 +3403,16 @@ cbm_daemon_runtime_cancel_result_t cbm_daemon_runtime_client_application_cancel( cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const void *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms) { + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms) { if (response_out) { *response_out = NULL; } if (response_length_out) { *response_length_out = 0; } + if (request_sent_out) { + *request_sent_out = false; + } if (!client || !response_out || !response_length_out || request_token == CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID || timeout_ms == CBM_DAEMON_IPC_WAIT_FOREVER || @@ -3465,6 +3468,9 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re request_can_send && cbm_daemon_ipc_send_frame(connection, CBM_DAEMON_FRAME_REQUEST, CBM_DAEMON_RUNTIME_OP_APPLICATION_REQUEST, wire, (uint32_t)wire_length); + if (request_sent_out) { + *request_sent_out = sent; + } bool send_cancel = false; cbm_mutex_lock(&client->state_mutex); client->application_request_sent = sent; @@ -3563,9 +3569,9 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re if (!cbm_daemon_runtime_client_application_token_reserve(client, &request_token)) { return CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; } - return cbm_daemon_runtime_client_application_request_tagged(client, request_token, request, - request_length, response_out, - response_length_out, timeout_ms); + return cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, request_length, response_out, response_length_out, NULL, + timeout_ms); } bool cbm_daemon_runtime_client_close_begin(cbm_daemon_runtime_client_t *client) { diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index 8a3152f06..fa0d5da9d 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -427,11 +427,12 @@ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_re /* Execute using the sole outstanding token returned by token_reserve(). This * is the cancellable frontend path; the legacy helper above reserves - * internally. */ + * internally. request_sent_out reports whether the application frame crossed + * the local transport, including failures while awaiting its response. */ cbm_daemon_runtime_application_status_t cbm_daemon_runtime_client_application_request_tagged( cbm_daemon_runtime_client_t *client, cbm_daemon_runtime_application_token_t request_token, const void *request, uint32_t request_length, uint8_t **response_out, - uint32_t *response_length_out, uint32_t timeout_ms); + uint32_t *response_length_out, bool *request_sent_out, uint32_t timeout_ms); /* Begin a two-phase close without freeing the client. This atomically rejects * future exchanges and interrupts an exchange already in flight. Exactly one diff --git a/src/foundation/index_policy.c b/src/foundation/index_policy.c index 0d2b986bf..80aa2f8a6 100644 --- a/src/foundation/index_policy.c +++ b/src/foundation/index_policy.c @@ -1,21 +1,38 @@ #include "foundation/index_policy.h" +#include #include #include -static const char *const INDEX_POLICY_KEYS[] = { - CBM_INDEX_CONFIG_MAX_FILES, - CBM_INDEX_CONFIG_MAX_SOURCE_MB, +typedef struct { + const char *key; + size_t field_offset; + uint64_t minimum; + uint64_t maximum; + uint64_t multiplier; +} index_policy_metadata_t; + +static const index_policy_metadata_t INDEX_POLICY_METADATA[] = { + {CBM_INDEX_CONFIG_MAX_FILES, offsetof(cbm_index_resource_policy_t, max_files), 1, + CBM_INDEX_MAX_FILES_VALUE, 1}, + {CBM_INDEX_CONFIG_MAX_SOURCE_MB, offsetof(cbm_index_resource_policy_t, max_source_bytes), 1, + CBM_INDEX_MAX_SOURCE_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_RSS_MB, offsetof(cbm_index_resource_policy_t, max_rss_bytes), + CBM_INDEX_MIN_RSS_MB_VALUE, CBM_INDEX_MAX_RSS_MB_VALUE, CBM_INDEX_MIB_BYTES}, + {CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, offsetof(cbm_index_resource_policy_t, max_duration_ms), + 1, CBM_INDEX_MAX_DURATION_SECONDS_VALUE, UINT64_C(1000)}, }; -static void set_error(char *error, size_t error_size, const char *key, uint64_t maximum) { +static void set_error(char *error, size_t error_size, const char *key, uint64_t minimum, + uint64_t maximum) { if (error && error_size > 0) { - (void)snprintf(error, error_size, "%s must be off or an integer from 1 to %llu", key, - (unsigned long long)maximum); + (void)snprintf(error, error_size, "%s must be off or an integer from %llu to %llu", key, + (unsigned long long)minimum, (unsigned long long)maximum); } } -static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t *parsed) { +static bool parse_bounded_uint64(const char *value, uint64_t minimum, uint64_t maximum, + uint64_t *parsed) { if (!value || !value[0] || !parsed) { return false; } @@ -30,7 +47,7 @@ static bool parse_bounded_uint64(const char *value, uint64_t maximum, uint64_t * } result = result * 10U + digit; } - if (result == 0) { + if (result < minimum) { return false; } *parsed = result; @@ -44,20 +61,39 @@ void cbm_index_policy_init(cbm_index_resource_policy_t *policy) { } bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy) { + if (!policy) { + return false; + } + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + + INDEX_POLICY_METADATA[index].field_offset); + if (limit->enabled) { + return true; + } + } + return false; +} + +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy) { return policy && (policy->max_files.enabled || policy->max_source_bytes.enabled); } +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy) { + return policy && (policy->max_rss_bytes.enabled || policy->max_duration_ms.enabled); +} + size_t cbm_index_policy_key_count(void) { - return sizeof(INDEX_POLICY_KEYS) / sizeof(INDEX_POLICY_KEYS[0]); + return sizeof(INDEX_POLICY_METADATA) / sizeof(INDEX_POLICY_METADATA[0]); } const char *cbm_index_policy_key_at(size_t index) { - return index < cbm_index_policy_key_count() ? INDEX_POLICY_KEYS[index] : NULL; + return index < cbm_index_policy_key_count() ? INDEX_POLICY_METADATA[index].key : NULL; } const char *cbm_index_policy_default_value(const char *key) { for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { - if (key && strcmp(key, INDEX_POLICY_KEYS[index]) == 0) { + if (key && strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { return "off"; } } @@ -70,34 +106,33 @@ bool cbm_index_policy_set(cbm_index_resource_policy_t *policy, const char *key, error[0] = '\0'; } if (!policy || !key || !value) { - set_error(error, error_size, key ? key : "index resource limit", 0); + set_error(error, error_size, key ? key : "index resource limit", 0, 0); return false; } - cbm_index_limit_u64_t *target = NULL; - uint64_t maximum = 0; - uint64_t multiplier = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - target = &policy->max_files; - maximum = CBM_INDEX_MAX_FILES_VALUE; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - target = &policy->max_source_bytes; - maximum = CBM_INDEX_MAX_SOURCE_MB_VALUE; - multiplier = CBM_INDEX_MIB_BYTES; - } else { - set_error(error, error_size, key, 0); + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { + set_error(error, error_size, key, 0, 0); return false; } + cbm_index_limit_u64_t *target = + (cbm_index_limit_u64_t *)((unsigned char *)policy + metadata->field_offset); cbm_index_limit_u64_t candidate = {0}; if (strcmp(value, "off") != 0) { uint64_t parsed = 0; - if (!parse_bounded_uint64(value, maximum, &parsed)) { - set_error(error, error_size, key, maximum); + if (!parse_bounded_uint64(value, metadata->minimum, metadata->maximum, &parsed)) { + set_error(error, error_size, key, metadata->minimum, metadata->maximum); return false; } candidate.enabled = true; - candidate.value = parsed * multiplier; + candidate.value = parsed * metadata->multiplier; } *target = candidate; return true; @@ -108,18 +143,21 @@ bool cbm_index_policy_format(const cbm_index_resource_policy_t *policy, const ch if (!policy || !key || !out || out_size == 0) { return false; } - const cbm_index_limit_u64_t *limit = NULL; - uint64_t divisor = 1; - if (strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0) { - limit = &policy->max_files; - } else if (strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0) { - limit = &policy->max_source_bytes; - divisor = CBM_INDEX_MIB_BYTES; - } else { + const index_policy_metadata_t *metadata = NULL; + for (size_t index = 0; index < cbm_index_policy_key_count(); index++) { + if (strcmp(key, INDEX_POLICY_METADATA[index].key) == 0) { + metadata = &INDEX_POLICY_METADATA[index]; + break; + } + } + if (!metadata) { return false; } + const cbm_index_limit_u64_t *limit = + (const cbm_index_limit_u64_t *)((const unsigned char *)policy + metadata->field_offset); int length = limit->enabled - ? snprintf(out, out_size, "%llu", (unsigned long long)(limit->value / divisor)) + ? snprintf(out, out_size, "%llu", + (unsigned long long)(limit->value / metadata->multiplier)) : snprintf(out, out_size, "off"); return length >= 0 && (size_t)length < out_size; } @@ -130,6 +168,10 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { return "files"; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return "source_bytes"; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return "rss_bytes"; + case CBM_INDEX_RESOURCE_DURATION_MS: + return "duration_ms"; case CBM_INDEX_RESOURCE_NONE: default: return "unknown"; @@ -137,7 +179,13 @@ const char *cbm_index_resource_name(cbm_index_resource_t resource) { } const char *cbm_index_resource_unit(cbm_index_resource_t resource) { - return resource == CBM_INDEX_RESOURCE_FILES ? "files" : "bytes"; + if (resource == CBM_INDEX_RESOURCE_FILES) { + return "files"; + } + if (resource == CBM_INDEX_RESOURCE_DURATION_MS) { + return "milliseconds"; + } + return "bytes"; } const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { @@ -146,6 +194,10 @@ const char *cbm_index_resource_config_key(cbm_index_resource_t resource) { return CBM_INDEX_CONFIG_MAX_FILES; case CBM_INDEX_RESOURCE_SOURCE_BYTES: return CBM_INDEX_CONFIG_MAX_SOURCE_MB; + case CBM_INDEX_RESOURCE_RSS_BYTES: + return CBM_INDEX_CONFIG_MAX_RSS_MB; + case CBM_INDEX_RESOURCE_DURATION_MS: + return CBM_INDEX_CONFIG_MAX_DURATION_SECONDS; case CBM_INDEX_RESOURCE_NONE: default: return "index_resource_limit"; diff --git a/src/foundation/index_policy.h b/src/foundation/index_policy.h index 6a6c499fc..617a6ab7d 100644 --- a/src/foundation/index_policy.h +++ b/src/foundation/index_policy.h @@ -7,9 +7,14 @@ #define CBM_INDEX_CONFIG_MAX_FILES "index_max_files" #define CBM_INDEX_CONFIG_MAX_SOURCE_MB "index_max_source_mb" +#define CBM_INDEX_CONFIG_MAX_RSS_MB "index_max_rss_mb" +#define CBM_INDEX_CONFIG_MAX_DURATION_SECONDS "index_max_duration_seconds" #define CBM_INDEX_MAX_FILES_VALUE UINT64_C(10000000) #define CBM_INDEX_MAX_SOURCE_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MIN_RSS_MB_VALUE UINT64_C(64) +#define CBM_INDEX_MAX_RSS_MB_VALUE UINT64_C(1048576) +#define CBM_INDEX_MAX_DURATION_SECONDS_VALUE UINT64_C(86400) #define CBM_INDEX_MIB_BYTES UINT64_C(1048576) typedef struct { @@ -20,12 +25,16 @@ typedef struct { typedef struct { cbm_index_limit_u64_t max_files; cbm_index_limit_u64_t max_source_bytes; + cbm_index_limit_u64_t max_rss_bytes; + cbm_index_limit_u64_t max_duration_ms; } cbm_index_resource_policy_t; typedef enum { CBM_INDEX_RESOURCE_NONE = 0, CBM_INDEX_RESOURCE_FILES, CBM_INDEX_RESOURCE_SOURCE_BYTES, + CBM_INDEX_RESOURCE_RSS_BYTES, + CBM_INDEX_RESOURCE_DURATION_MS, } cbm_index_resource_t; typedef struct { @@ -36,6 +45,8 @@ typedef struct { void cbm_index_policy_init(cbm_index_resource_policy_t *policy); bool cbm_index_policy_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_discovery_enabled(const cbm_index_resource_policy_t *policy); +bool cbm_index_policy_worker_enabled(const cbm_index_resource_policy_t *policy); size_t cbm_index_policy_key_count(void); const char *cbm_index_policy_key_at(size_t index); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 6ce3615b0..37360d831 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -14,6 +14,7 @@ #include "platform.h" /* cbm_now_ms */ #include "sanitized.h" /* CBM_SANITIZED — spawn-retry budget */ +#include #include #include #include @@ -21,15 +22,18 @@ #ifdef _WIN32 #include +#include #include "win_utf8.h" /* cbm_utf8_to_wide — spawn the worker with a wide command line so a * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#include #include #include #include #ifdef __APPLE__ #include +#include #include #endif extern char **environ; @@ -456,6 +460,546 @@ struct cbm_subprocess { #endif }; +static void cbm_rss_add_saturated(uint64_t *total, uint64_t value) { + if (value > UINT64_MAX - *total) { + *total = UINT64_MAX; + } else { + *total += value; + } +} + +#ifdef CBM_ENABLE_TEST_SEAMS +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count) { + uint64_t total = 0; + for (size_t index = 0; values && index < count; index++) { + cbm_rss_add_saturated(&total, values[index]); + } + return total; +} + +static atomic_int g_tree_rss_member_fault; + +void cbm_subprocess_tree_rss_fault_members_for_testing(int platform_error) { + atomic_store_explicit(&g_tree_rss_member_fault, platform_error, memory_order_release); +} + +/* Test builds can make every NON-root member's measurement fail with one + * platform error (errno on POSIX, GetLastError() on Windows); 0 = real call. + * Production builds compile the check out entirely. */ +static int cbm_tree_rss_member_fault(bool root) { + return root ? 0 : atomic_load_explicit(&g_tree_rss_member_fault, memory_order_acquire); +} +#endif + +/* A member that cannot be measured is skipped only when the platform says it + * no longer exists (it exited between the listing and the measurement, and + * holds nothing). Any other failure is a live member whose memory a total + * would silently leave out, so the probe fails instead -- a watchdog that + * under-reports is one that quietly stopped watching. The root is strict + * either way. */ +#ifdef _WIN32 +static HANDLE cbm_tree_rss_open_member(DWORD pid, bool root, DWORD *error_out) { +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + *error_out = (DWORD)fault; + return NULL; + } +#else + (void)root; +#endif + /* GetProcessMemoryInfo documents PROCESS_QUERY_INFORMATION | + * PROCESS_VM_READ. PROCESS_QUERY_LIMITED_INFORMATION is a lesser + * alternative on newer Windows; the Job Object membership list + * already closes most of the pid-reuse window between listing and + * opening. */ + HANDLE member = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); + *error_out = member ? ERROR_SUCCESS : GetLastError(); + return member; +} + +static bool cbm_tree_rss_member_exited(HANDLE member) { + DWORD exit_code = 0; + return GetExitCodeProcess(member, &exit_code) && exit_code != STILL_ACTIVE; +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DWORD capacity = 32; + JOBOBJECT_BASIC_PROCESS_ID_LIST *processes = NULL; + for (;;) { + size_t bytes = sizeof(*processes) + (size_t)(capacity - 1) * sizeof(ULONG_PTR); + if (bytes > UINT32_MAX) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + JOBOBJECT_BASIC_PROCESS_ID_LIST *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, processes, bytes); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + processes = grown; + ZeroMemory(processes, bytes); + processes->NumberOfAssignedProcesses = capacity; + if (QueryInformationJobObject(process->job, JobObjectBasicProcessIdList, processes, + (DWORD)bytes, NULL)) { + break; + } + if (GetLastError() != ERROR_MORE_DATA || capacity > (UINT32_MAX / 2U)) { + cbm_free(CBM_MEM_CLASS_OTHER, processes); + return CBM_PROC_TREE_RSS_ERROR; + } + capacity *= 2U; + } + + uint64_t total = 0; + DWORD measured = 0; + bool failed = false; + for (DWORD index = 0; index < processes->NumberOfProcessIdsInList && !failed; index++) { + DWORD pid = (DWORD)processes->ProcessIdList[index]; + bool root = pid == process->process_id; + DWORD open_error = ERROR_SUCCESS; + HANDLE member = cbm_tree_rss_open_member(pid, root, &open_error); + if (!member) { + /* ERROR_INVALID_PARAMETER: no process has this id any more. */ + failed = root || open_error != ERROR_INVALID_PARAMETER; + continue; + } + PROCESS_MEMORY_COUNTERS memory; + ZeroMemory(&memory, sizeof(memory)); + if (GetProcessMemoryInfo(member, &memory, sizeof(memory))) { + cbm_rss_add_saturated(&total, (uint64_t)memory.WorkingSetSize); + measured++; + } else { + failed = root || !cbm_tree_rss_member_exited(member); + } + CloseHandle(member); + } + DWORD listed = processes->NumberOfProcessIdsInList; + cbm_free(CBM_MEM_CLASS_OTHER, processes); + if (failed || (measured == 0 && listed > 0)) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#elif defined(__APPLE__) +/* The members of the contained process group, as the kernel lists them. An + * all-process scan cannot decide membership: proc_pidinfo answers EPERM for + * every process of another user, so "not in our group" and "in our group but + * unreadable" would look the same. Returns the member count, or -1. */ +static int cbm_tree_rss_list_pids(uint32_t type, uint32_t arg, pid_t **pids_out) { + int capacity = 64; + pid_t *pids = NULL; + for (;;) { + pid_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, pids, (size_t)capacity * sizeof(*pids)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return -1; + } + pids = grown; + errno = 0; + int bytes = proc_listpids(type, arg, pids, capacity * (int)sizeof(*pids)); + if (bytes < 0 || (bytes == 0 && errno != 0)) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return -1; + } + int count = bytes / (int)sizeof(*pids); + if (count < capacity) { + *pids_out = pids; + return count; + } + if (capacity > INT_MAX / (int)sizeof(*pids) / 2) { + cbm_free(CBM_MEM_CLASS_OTHER, pids); + return -1; + } + capacity *= 2; + } +} + +/* Bound on the processes one probe retains; past it the probe fails closed. */ +enum { CBM_APPLE_PROBE_MAX_PROCS = 65536 }; + +/* The worker's process group plus every descendant by parent pid. Children + * placed in a group of their own (git discovery) are not in the group list. + * Pids already collected are skipped, so nothing is counted twice. Returns the + * count, or -1. */ +static int cbm_tree_rss_list_tree(pid_t pgid, pid_t **pids_out) { + pid_t *tree = NULL; + int count = cbm_tree_rss_list_pids(PROC_PGRP_ONLY, (uint32_t)pgid, &tree); + if (count < 0) { + return -1; + } + int capacity = count > 0 ? count : 1; + for (int index = 0; index < count; index++) { + pid_t *children = NULL; + int child_count = tree[index] > 0 ? cbm_tree_rss_list_pids(PROC_PPID_ONLY, + (uint32_t)tree[index], &children) + : 0; + if (child_count < 0) { + /* ESRCH-style races surface as an empty listing; a real error + * must not under-report the tree. */ + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + for (int child = 0; child < child_count; child++) { + bool known = children[child] <= 0; + for (int seen = 0; !known && seen < count; seen++) { + known = tree[seen] == children[child]; + } + if (known) { + continue; + } + if (count >= CBM_APPLE_PROBE_MAX_PROCS) { + cbm_free(CBM_MEM_CLASS_OTHER, children); + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + if (count == capacity) { + pid_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, tree, (size_t)capacity * 2 * sizeof(*tree)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, children); + cbm_free(CBM_MEM_CLASS_OTHER, tree); + return -1; + } + tree = grown; + capacity *= 2; + } + tree[count++] = children[child]; + } + cbm_free(CBM_MEM_CLASS_OTHER, children); + } + *pids_out = tree; + return count; +} + +static int cbm_tree_rss_member_rusage(pid_t pid, bool root, struct rusage_info_v2 *usage) { +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + errno = fault; + return -1; + } +#else + (void)root; +#endif + return proc_pid_rusage(pid, RUSAGE_INFO_V2, (rusage_info_t *)usage); +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + pid_t *pids = NULL; + int count = cbm_tree_rss_list_tree(process->pgid, &pids); + if (count < 0) { + return CBM_PROC_TREE_RSS_ERROR; + } + + uint64_t total = 0; + int measured = 0; + bool failed = false; + for (int index = 0; index < count && !failed; index++) { + if (pids[index] <= 0) { + continue; + } + bool root = pids[index] == process->pid; + struct rusage_info_v2 usage; + if (cbm_tree_rss_member_rusage(pids[index], root, &usage) != 0) { + /* ESRCH: the member exited after the listing. */ + failed = root || errno != ESRCH; + continue; + } + /* phys_footprint is what the OS charges, matching cbm_mem_charged(). + * ri_resident_size still counts pages mimalloc already marked + * MADV_FREE_REUSABLE, so a Mac worker can look like 17 GB RSS while + * the kernel holds it to 5.5 GB. */ + cbm_rss_add_saturated(&total, usage.ri_phys_footprint); + measured++; + } + cbm_free(CBM_MEM_CLASS_OTHER, pids); + if (failed) { + return CBM_PROC_TREE_RSS_ERROR; + } + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#else +static bool cbm_proc_pid_name(const char *name) { + if (!name || !name[0]) { + return false; + } + for (const unsigned char *cursor = (const unsigned char *)name; *cursor; cursor++) { + if (*cursor < '0' || *cursor > '9') { + return false; + } + } + return true; +} + +typedef enum { + CBM_LINUX_STAT_OK = 0, + CBM_LINUX_STAT_GONE, /* the process exited after the /proc listing */ + CBM_LINUX_STAT_UNREADABLE, /* a process whose stat could not be read or parsed */ +} cbm_linux_stat_t; + +/* /proc//stat is world-readable, so on a default /proc mount the only + * expected failure is the exit race: ENOENT at open, ESRCH at read. Anything + * else (EACCES under hidepid=1, a parse failure) leaves the process's group + * unknown, and the probe fails closed rather than guess it is not ours. */ +static cbm_linux_stat_t cbm_linux_stat_failure(int error) { + return error == ENOENT || error == ESRCH ? CBM_LINUX_STAT_GONE : CBM_LINUX_STAT_UNREADABLE; +} + +static cbm_linux_stat_t cbm_linux_proc_stat_line(const char *pid_name, bool root, char *line, + size_t line_size) { + char path[64]; + int length = snprintf(path, sizeof(path), "/proc/%s/stat", pid_name); + if (length <= 0 || length >= (int)sizeof(path)) { + return CBM_LINUX_STAT_UNREADABLE; + } +#ifdef CBM_ENABLE_TEST_SEAMS + int fault = cbm_tree_rss_member_fault(root); + if (fault != 0) { + return cbm_linux_stat_failure(fault); + } +#else + (void)root; +#endif + errno = 0; + FILE *stat_file = cbm_fopen(path, "rb"); + if (!stat_file) { + return cbm_linux_stat_failure(errno); + } + errno = 0; + bool read = fgets(line, (int)line_size, stat_file) != NULL; + int read_error = errno; + (void)fclose(stat_file); + if (!read) { + return read_error == ESRCH ? CBM_LINUX_STAT_GONE : CBM_LINUX_STAT_UNREADABLE; + } + return CBM_LINUX_STAT_OK; +} + +static cbm_linux_stat_t cbm_linux_proc_stat(const char *pid_name, bool root, pid_t *parent, + pid_t *group, int64_t *rss_pages) { + char stat_line[4096]; + cbm_linux_stat_t status = + cbm_linux_proc_stat_line(pid_name, root, stat_line, sizeof(stat_line)); + if (status != CBM_LINUX_STAT_OK) { + return status; + } + char *command_end = strrchr(stat_line, ')'); + if (!command_end || command_end[1] != ' ') { + return CBM_LINUX_STAT_UNREADABLE; + } + + char *save = NULL; + char *token = strtok_r(command_end + 2, " ", &save); + int field = 3; + bool have_parent = false; + bool have_group = false; + bool have_rss = false; + while (token) { + if (field == 4 || field == 5 || field == 24) { + char *end = NULL; + long long value = strtoll(token, &end, 10); + if (!end || *end != '\0') { + return CBM_LINUX_STAT_UNREADABLE; + } + if (field == 4) { + *parent = (pid_t)value; + have_parent = true; + } else if (field == 5) { + *group = (pid_t)value; + have_group = true; + } else { + *rss_pages = (int64_t)value; + have_rss = true; + break; + } + } + token = strtok_r(NULL, " ", &save); + field++; + } + return have_parent && have_group && have_rss ? CBM_LINUX_STAT_OK : CBM_LINUX_STAT_UNREADABLE; +} + +/* One scanned process. `counted` is set once it is known to belong to the + * worker tree. */ +typedef struct { + pid_t pid; + pid_t parent; + pid_t group; + uint64_t bytes; + bool counted; +} cbm_linux_proc_t; + +/* Bound on the processes one probe retains. Past it the probe fails closed + * instead of reporting a total that may miss part of the tree. */ +enum { CBM_LINUX_PROBE_MAX_PROCS = 262144, CBM_LINUX_PROBE_INITIAL_PROCS = 512 }; + +static bool cbm_linux_probe_push(cbm_linux_proc_t **procs, size_t *count, size_t *capacity, + const cbm_linux_proc_t *proc) { + if (*count == *capacity) { + if (*capacity >= CBM_LINUX_PROBE_MAX_PROCS) { + return false; + } + size_t grown_capacity = *capacity ? *capacity * 2 : CBM_LINUX_PROBE_INITIAL_PROCS; + cbm_linux_proc_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, *procs, grown_capacity * sizeof(**procs)); + if (!grown) { + return false; + } + *procs = grown; + *capacity = grown_capacity; + } + (*procs)[(*count)++] = *proc; + return true; +} + +static int cbm_linux_probe_by_pid(const void *left, const void *right) { + pid_t a = ((const cbm_linux_proc_t *)left)->pid; + pid_t b = ((const cbm_linux_proc_t *)right)->pid; + return (a > b) - (a < b); +} + +/* Mark every process of the worker tree: members of its process group, plus + * any descendant by parent pid. Descendants can sit in a group of their own + * (git discovery children are spawned with setpgid), so the group alone would + * miss them. `procs` is sorted by pid, so a parent is found by binary search + * and each pass is O(n log n); passes repeat until no process is added (at + * most the tree depth). A process is counted once because the flag is checked + * before it is set. */ +static void cbm_linux_probe_mark_tree(cbm_linux_proc_t *procs, size_t count, pid_t pgid) { + qsort(procs, count, sizeof(*procs), cbm_linux_probe_by_pid); + for (size_t index = 0; index < count; index++) { + procs[index].counted = procs[index].group == pgid; + } + bool changed = true; + while (changed) { + changed = false; + for (size_t index = 0; index < count; index++) { + if (procs[index].counted || procs[index].parent <= 1) { + continue; + } + cbm_linux_proc_t key = {.pid = procs[index].parent}; + const cbm_linux_proc_t *parent = + bsearch(&key, procs, count, sizeof(*procs), cbm_linux_probe_by_pid); + if (parent && parent->counted) { + procs[index].counted = true; + changed = true; + } + } + } +} + +static cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_platform(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + DIR *proc = opendir("/proc"); + if (!proc) { + return CBM_PROC_TREE_RSS_ERROR; + } + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + (void)closedir(proc); + return CBM_PROC_TREE_RSS_ERROR; + } + + cbm_linux_proc_t *procs = NULL; + size_t count = 0; + size_t capacity = 0; + bool failed = false; + struct dirent *entry; + while (!failed && (entry = readdir(proc)) != NULL) { + if (!cbm_proc_pid_name(entry->d_name)) { + continue; + } + pid_t parent = 0; + pid_t group = 0; + int64_t pages = 0; + long pid = strtol(entry->d_name, NULL, 10); + bool root_entry = pid == (long)process->pid; + cbm_linux_stat_t status = + cbm_linux_proc_stat(entry->d_name, root_entry, &parent, &group, &pages); + if (status != CBM_LINUX_STAT_OK) { + failed = root_entry || status != CBM_LINUX_STAT_GONE; + continue; + } + if (pages < 0) { + continue; + } + uint64_t page_count = (uint64_t)pages; + cbm_linux_proc_t item = {.pid = (pid_t)pid, + .parent = parent, + .group = group, + .bytes = page_count > UINT64_MAX / (uint64_t)page_size + ? UINT64_MAX + : page_count * (uint64_t)page_size}; + failed = !cbm_linux_probe_push(&procs, &count, &capacity, &item); + } + (void)closedir(proc); + if (failed) { + cbm_free(CBM_MEM_CLASS_OTHER, procs); + return CBM_PROC_TREE_RSS_ERROR; + } + if (!procs || count == 0) { + cbm_free(CBM_MEM_CLASS_OTHER, procs); + return CBM_PROC_TREE_RSS_EMPTY; + } + + cbm_linux_probe_mark_tree(procs, count, process->pgid); + uint64_t total = 0; + int measured = 0; + for (size_t index = 0; index < count; index++) { + if (procs[index].counted) { + cbm_rss_add_saturated(&total, procs[index].bytes); + measured++; + } + } + cbm_free(CBM_MEM_CLASS_OTHER, procs); + if (measured == 0) { + return CBM_PROC_TREE_RSS_EMPTY; + } + *rss_bytes = total; + return CBM_PROC_TREE_RSS_OK; +} +#endif + +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes) { + if (!process || !rss_bytes) { + return CBM_PROC_TREE_RSS_ERROR; + } + *rss_bytes = 0; + return cbm_subprocess_tree_rss_platform(process, rss_bytes); +} + +bool cbm_subprocess_root_running(const cbm_subprocess_t *process) { + if (!process || process->root_reaped) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process) { + return process && process->termination_started; +} + +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process) { + if (!process) { + return false; + } + int lifecycle = atomic_load_explicit(&process->lifecycle, memory_order_acquire); + return lifecycle == CBM_SUBPROCESS_ACTIVE || lifecycle == CBM_SUBPROCESS_CANCEL_REQUESTED; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 4dc8f1870..a5b44474a 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -22,6 +22,7 @@ #include #include /* size_t (cbm_build_win_cmdline) */ +#include /* How a supervised child ended. */ typedef enum { @@ -118,6 +119,12 @@ typedef enum { CBM_PROC_POLL_TERMINAL = 1 } cbm_proc_poll_t; +typedef enum { + CBM_PROC_TREE_RSS_ERROR = -1, + CBM_PROC_TREE_RSS_EMPTY = 0, + CBM_PROC_TREE_RSS_OK = 1, +} cbm_proc_tree_rss_status_t; + /* Spawn opts->bin and return immediately with a supervisor handle. On success, * *out owns the process until a terminal poll followed by destroy. Spawn copies * the option strings/argv it needs after return; log_ud remains caller-owned until @@ -148,6 +155,19 @@ int cbm_subprocess_spawn(const cbm_proc_opts_t *opts, cbm_subprocess_t **out); * tree when cancel_grace_ms elapses. Callers must keep polling to make progress. */ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t *out); +/* Read the current charged memory of the complete contained process tree. + * On macOS this is phys_footprint (what the OS holds against the process); + * elsewhere it is RSS. OK returns an overflow-safe byte total, EMPTY means + * the owned tree currently has no observable members, and ERROR means no + * trustworthy measurement could be obtained. A member that exits between the + * listing and its measurement is ignored (the platform reports it gone); any + * other member that cannot be measured is ERROR, never a smaller total. */ +cbm_proc_tree_rss_status_t cbm_subprocess_tree_rss_bytes(cbm_subprocess_t *process, + uint64_t *rss_bytes); +bool cbm_subprocess_root_running(const cbm_subprocess_t *process); +bool cbm_subprocess_termination_pending(const cbm_subprocess_t *process); +bool cbm_subprocess_supervision_active(const cbm_subprocess_t *process); + /* Record an explicit cancellation request without waiting. Safe to repeat and * safe to call from a cancellation thread while one owner thread polls. The * owner must stop cancellation producers before destroying the handle. true @@ -223,6 +243,11 @@ void cbm_subprocess_hold_parent_until_child_exits_for_testing(int spawns); * shows for a child that is EXITING: setpgid and getpgid answer ESRCH while * waitid does not report it yet. Test builds only. */ void cbm_subprocess_observe_exiting_child_for_testing(int spawns); +uint64_t cbm_subprocess_rss_sum_for_testing(const uint64_t *values, size_t count); +/* Make the tree probe's measurement of every NON-root member fail with + * platform_error (errno on POSIX, GetLastError() on Windows), as a live or an + * exiting member would; 0 restores the real calls. Test builds only. */ +void cbm_subprocess_tree_rss_fault_members_for_testing(int platform_error); #endif /* How the POSIX fork+exec child closed its inherited descriptors (#1484). */ diff --git a/src/main.c b/src/main.c index 08687c30a..858822b78 100644 --- a/src/main.c +++ b/src/main.c @@ -887,6 +887,8 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo const char *worker_marker = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_MARKER_ARG); const char *worker_quarantine = cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_QUARANTINE_ARG); + const char *worker_publish_fence = + cli_strip_flag_value(&argc, argv, CBM_INDEX_WORKER_PUBLISH_FENCE_ARG); if (index_worker) { /* The graph lives on this process's heaps: SQLite gets heaps of its * own here, and only here (cbm_sqlite_dedicated_heap). */ @@ -902,6 +904,7 @@ static int run_cli(int argc, char **argv, cbm_project_lock_manager_t *project_lo cbm_index_set_worker_role_options(index_worker, response_out, worker_single_thread, worker_marker, worker_quarantine, cbm_index_worker_memory_budget_bytes()); + cbm_index_set_worker_publish_fence(index_worker ? worker_publish_fence : NULL); if (argc < MAIN_MIN_ARGC) { (void)fprintf(stderr, CLI_USAGE); @@ -3601,6 +3604,7 @@ int main(int argc, char **argv) { cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, invocation.memory_budget_bytes); + cbm_index_set_worker_publish_fence(invocation.publish_fence_file); #ifndef _WIN32 /* Split into three ordered steps rather than one condition, because the * ORDER is load-bearing and the middle step only exists in test builds: @@ -3779,7 +3783,6 @@ int main(int argc, char **argv) { cbm_daemon_bootstrap_result_t bootstrap_result; cbm_daemon_bootstrap_status_t bootstrap_status = main_client_bootstrap_with_upgrade(&bootstrap_config, &bootstrap_result, true); - cbm_daemon_ipc_endpoint_free(endpoint); if (bootstrap_status != CBM_DAEMON_BOOTSTRAP_CONNECTED || !bootstrap_result.client) { main_report_client_bootstrap_failure(&bootstrap_result); (void)main_version_cohort_close(&client_cohort_lease, &client_cohort_manager); diff --git a/src/operations/index.c b/src/operations/index.c index cdc3ef304..dc2c8b1d1 100644 --- a/src/operations/index.c +++ b/src/operations/index.c @@ -42,6 +42,11 @@ static char *index_strdup(const char *text) { return copy; } +/* One release point for project-name strings from cbm_project_name_from_path. */ +static void index_release_text(char *text) { + free(text); +} + static char *index_string_arg(const char *args_json, const char *key) { const char *json = args_json ? args_json : "{}"; yyjson_doc *doc = yyjson_read(json, strlen(json), 0); @@ -250,9 +255,8 @@ bool cbm_index_operation_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val return valid && yyjson_mut_obj_add_val(doc, root, INDEX_WORKER_POLICY_KEY, encoded); } -static bool index_policy_from_worker_args(const char *args_json, - cbm_index_resource_policy_t *policy, char *error, - size_t error_size) { +bool cbm_index_policy_from_internal_args(const char *args_json, cbm_index_resource_policy_t *policy, + char *error, size_t error_size) { yyjson_doc *doc = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *encoded = @@ -283,7 +287,7 @@ static bool index_policy_from_worker_args(const char *args_json, static bool index_load_policy(const char *args_json, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (cbm_index_worker_active()) { - return index_policy_from_worker_args(args_json, policy, error, error_size); + return cbm_index_policy_from_internal_args(args_json, policy, error, error_size); } cbm_config_t *config = cbm_config_open(cbm_resolve_cache_dir()); bool loaded = cbm_config_load_index_policy(config, policy, error, error_size); @@ -442,6 +446,94 @@ static bool index_project_db_is_servable(const char *project, const char *db_pat return servable; } +/* Whether the project named by internal worker args has a servable index now. */ +static bool index_args_serving_index_servable(const char *args_json) { + char *repo_path = index_string_arg(args_json, "repo_path"); + char *name = index_string_arg(args_json, "name"); + char *project = cbm_project_name_from_path(name && name[0] ? name : repo_path); + bool servable = false; + char db_path[CBM_SZ_1K] = {0}; + if (project && index_db_path(project, db_path, sizeof(db_path))) { + servable = index_project_db_is_servable(project, db_path); + } + index_release_text(project); + cbm_operation_arg_free(name); + cbm_operation_arg_free(repo_path); + return servable; +} + +void cbm_index_request_facts_begin(const char *args_json, cbm_index_request_facts_t *facts) { + if (!facts) { + return; + } + facts->serving_index_existed = index_args_serving_index_servable(args_json); + facts->publication_committed = false; +} + +void cbm_index_request_facts_record(cbm_index_request_facts_t *facts, + const cbm_index_worker_result_t *worker_result) { + if (facts && worker_result && worker_result->publication_committed) { + facts->publication_committed = true; + } +} + +char *cbm_index_worker_resource_response(const char *args_json, + const cbm_index_worker_result_t *worker_result, + const cbm_index_request_facts_t *facts) { + if (!worker_result) { + return NULL; + } + /* The publication fence makes a limit-stopped attempt and a publication by + * that attempt mutually exclusive; an EARLIER attempt of the request may + * still have published (it crashed after committing), and an index that + * appeared during the request (artifact bootstrap) was not preserved. */ + bool serving_index_preserved = + facts && facts->serving_index_existed && !facts->publication_committed && + !worker_result->publication_committed && index_args_serving_index_servable(args_json); + + const cbm_index_resource_violation_t *violation = &worker_result->resource_violation; + const char *config_key = cbm_index_resource_config_key(violation->resource); + char message[CBM_SZ_256]; + (void)snprintf(message, sizeof(message), + worker_result->resource_probe_failed + ? "Worker resource measurement failed for %s" + : "Index worker exceeded %s", + config_key); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (!root) { + if (doc) { + yyjson_mut_doc_free(doc); + } + return NULL; + } + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "code", + worker_result->resource_probe_failed ? "resource_probe_failed" + : "resource_limit_exceeded"); + yyjson_mut_obj_add_str(doc, root, "stage", "worker"); + yyjson_mut_obj_add_str(doc, root, "resource", cbm_index_resource_name(violation->resource)); + if (!worker_result->resource_probe_failed) { + yyjson_mut_obj_add_uint(doc, root, "observed", violation->observed); + yyjson_mut_obj_add_uint(doc, root, "limit", violation->limit); + yyjson_mut_obj_add_str(doc, root, "unit", cbm_index_resource_unit(violation->resource)); + } + yyjson_mut_obj_add_bool(doc, root, "retryable", true); + yyjson_mut_obj_add_bool(doc, root, "serving_index_preserved", serving_index_preserved); + yyjson_mut_obj_add_strcpy(doc, root, "message", message); + char *repo_path = index_string_arg(args_json, "repo_path"); + if (repo_path) { + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo_path); + } + cbm_operation_arg_free(repo_path); + cbm_operation_result_t error = cbm_operation_result_take(cbm_operation_json_write(doc), true); + yyjson_mut_doc_free(doc); + char *wire = cbm_operation_result_wire_encode(&error); + cbm_operation_result_dispose(&error); + return wire; +} + static bool index_project_has_adr(cbm_store_t *store, const char *project, const char *root_path) { cbm_adr_t adr = {0}; if (store && project && cbm_store_adr_get(store, project, &adr) == CBM_STORE_OK) { @@ -909,6 +1001,11 @@ static char *index_encode_run_response(const char *project, const char *repo_pat return payload; } +static bool index_worker_publish_gate(void *context) { + (void)context; + return cbm_index_worker_claim_publication(); +} + static cbm_operation_result_t index_run_physical(const char *repo_path, const char *args_json, const cbm_operation_runtime_t *runtime) { char *mode_text = index_string_arg(args_json, "mode"); @@ -926,7 +1023,7 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch cbm_operation_arg_free(mode_text); cbm_operation_arg_free(name); cbm_operation_arg_free(metrics_out); - free(mutation_project); + index_release_text(mutation_project); return index_text_error("index operation blocked by another mutation for this project"); } if (runtime->cancelled && runtime->cancelled(runtime->cancelled_context)) { @@ -936,7 +1033,7 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch cbm_operation_arg_free(mode_text); cbm_operation_arg_free(name); cbm_operation_arg_free(metrics_out); - free(mutation_project); + index_release_text(mutation_project); return index_text_error("index operation cancelled for this request"); } cbm_index_mode_t mode = CBM_MODE_FULL; @@ -953,7 +1050,7 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch runtime->mutation_end(runtime->mutation_context, mutation_project); cbm_operation_arg_free(name); cbm_operation_arg_free(metrics_out); - free(mutation_project); + index_release_text(mutation_project); return index_text_error("failed to create pipeline"); } if (name && name[0] && !cbm_pipeline_set_project_name(pipeline, name)) { @@ -961,22 +1058,27 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch runtime->mutation_end(runtime->mutation_context, mutation_project); cbm_operation_arg_free(name); cbm_operation_arg_free(metrics_out); - free(mutation_project); + index_release_text(mutation_project); return index_text_error("invalid project name"); } cbm_operation_arg_free(name); cbm_pipeline_set_persistence(pipeline, persistence); cbm_index_resource_policy_t resource_policy; char policy_error[CBM_SZ_256] = {0}; - if (!index_policy_from_worker_args(args_json, &resource_policy, policy_error, - sizeof(policy_error))) { + if (!cbm_index_policy_from_internal_args(args_json, &resource_policy, policy_error, + sizeof(policy_error))) { cbm_pipeline_free(pipeline); runtime->mutation_end(runtime->mutation_context, mutation_project); cbm_operation_arg_free(metrics_out); - free(mutation_project); + index_release_text(mutation_project); return index_text_error(policy_error); } cbm_pipeline_set_resource_policy(pipeline, &resource_policy); + if (cbm_index_worker_publish_fence()) { + /* Supervised worker with a worker limit: claim publication from the + * supervisor before the serving database is replaced. */ + cbm_pipeline_set_publish_gate(pipeline, index_worker_publish_gate, NULL); + } index_doc_candidates_apply(pipeline, index_doc_candidates_from_args(args_json)); char *project = index_strdup(cbm_pipeline_project_name(pipeline)); index_try_artifact_bootstrap(project, repo_path); @@ -1017,8 +1119,8 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch else cbm_pipeline_free(pipeline); runtime->mutation_end(runtime->mutation_context, mutation_project); - free(project); - free(mutation_project); + index_release_text(project); + index_release_text(mutation_project); return payload ? cbm_operation_result_take(payload, rc != 0) : index_text_error("result encoding failed"); } @@ -1303,22 +1405,22 @@ cbm_operation_result_t cbm_index_operation_execute(const char *args_json, char *project = raw_name && raw_name[0] ? cbm_project_name_sanitize(raw_name) : NULL; cbm_operation_arg_free(raw_name); if (!project || !project[0]) { - free(project); + index_release_text(project); project = index_root_project(repo_path, owner_error, sizeof(owner_error)); } if (!project || !cbm_validate_project_name(project)) { - free(project); + index_release_text(project); free(repo_path); return index_text_error(owner_error[0] ? owner_error : "invalid index project name"); } if (status_mode) { cbm_operation_result_t out = runtime->index_status(runtime->index_status_context, project); - free(project); + index_release_text(project); free(repo_path); return out; } char *named_args = index_args_with_string(json, "name", project); - free(project); + index_release_text(project); if (!named_args) { free(repo_path); return index_text_error("failed to prepare named index request"); diff --git a/src/operations/index.h b/src/operations/index.h index 568a4a272..4d360124e 100644 --- a/src/operations/index.h +++ b/src/operations/index.h @@ -2,6 +2,7 @@ #define CBM_OPERATIONS_INDEX_H #include "foundation/index_policy.h" +#include "operations/index_supervisor.h" #include "operations/operation.h" #include @@ -21,6 +22,34 @@ bool cbm_index_operation_policy_add_to_args(yyjson_mut_doc *doc, yyjson_mut_val *root, const cbm_index_resource_policy_t *policy); +/* Decode the trusted policy a supervisor encoded on an internal worker + * request. False (with error filled) when it is missing or incomplete. */ +bool cbm_index_policy_from_internal_args(const char *args_json, cbm_index_resource_policy_t *policy, + char *error, size_t error_size); + +/* Request-scoped facts a worker resource failure is reported against. One + * supervised request can run several worker attempts (crash/hang recovery), and + * serving_index_preserved describes the request, not only its last attempt: + * it is true only when a servable index served the project when the request + * began AND no attempt of the request committed to replacing it (the + * publication fence, index_supervisor.h) AND that index is still servable. */ +typedef struct { + bool serving_index_existed; + bool publication_committed; +} cbm_index_request_facts_t; + +/* Capture the facts at the start of a request, before its first worker. */ +void cbm_index_request_facts_begin(const char *args_json, cbm_index_request_facts_t *facts); +/* Fold one finished attempt into the request facts. */ +void cbm_index_request_facts_record(cbm_index_request_facts_t *facts, + const cbm_index_worker_result_t *worker_result); +/* Native result wire (is_error set, code resource_limit_exceeded or + * resource_probe_failed) for a worker stopped by a resource limit; caller + * frees. NULL facts claim nothing: serving_index_preserved is then false. */ +char *cbm_index_worker_resource_response(const char *args_json, + const cbm_index_worker_result_t *worker_result, + const cbm_index_request_facts_t *facts); + /* The fork's key-value config store (defined in cli/cli.h). Declared here so * the encoder below can name it without pulling cli.h into every includer. */ struct cbm_config; diff --git a/src/operations/index_supervisor.c b/src/operations/index_supervisor.c index 0c6c2663e..8c13fe7ad 100644 --- a/src/operations/index_supervisor.c +++ b/src/operations/index_supervisor.c @@ -92,6 +92,63 @@ size_t cbm_index_worker_memory_budget_bytes(void) { return g_worker_memory_budget_bytes; } +/* ── Publication fence (see index_supervisor.h) ───────────────────── */ + +typedef enum { + INDEX_PUBLICATION_CLAIMED = 0, /* this caller created the fence file */ + INDEX_PUBLICATION_TAKEN, /* the other side created it first */ + INDEX_PUBLICATION_CLAIM_ERROR, /* neither could be established */ +} index_publication_claim_t; + +/* Exclusive creation is the single atomic decision both processes race for: + * "x" is O_CREAT|O_EXCL on POSIX and CREATE_NEW on Windows, so exactly one + * creator succeeds and the other gets EEXIST. A deletion-based claim would + * be ambiguous on Windows, where a delete-pending file reports access denied. */ +static index_publication_claim_t index_publication_claim(const char *fence_file) { + if (!fence_file || !fence_file[0]) { + return INDEX_PUBLICATION_CLAIM_ERROR; + } + errno = 0; + FILE *claim = cbm_fopen(fence_file, "wbx"); + if (claim) { + (void)fclose(claim); + return INDEX_PUBLICATION_CLAIMED; + } + return errno == EEXIST ? INDEX_PUBLICATION_TAKEN : INDEX_PUBLICATION_CLAIM_ERROR; +} + +static char g_worker_publish_fence[CBM_SZ_4K] = {0}; +static bool g_worker_publication_claimed = false; + +void cbm_index_set_worker_publish_fence(const char *fence_file) { + if (fence_file && fence_file[0]) { + (void)snprintf(g_worker_publish_fence, sizeof(g_worker_publish_fence), "%s", fence_file); + } else { + g_worker_publish_fence[0] = '\0'; + } + g_worker_publication_claimed = false; +} + +const char *cbm_index_worker_publish_fence(void) { + return g_worker_publish_fence[0] ? g_worker_publish_fence : NULL; +} + +bool cbm_index_worker_claim_publication(void) { + if (!g_worker_publish_fence[0] || g_worker_publication_claimed) { + return true; + } + index_publication_claim_t claim = index_publication_claim(g_worker_publish_fence); + if (claim == INDEX_PUBLICATION_CLAIMED) { + g_worker_publication_claimed = true; + cbm_log_info("index.worker.publication_claimed", "fence", g_worker_publish_fence); + return true; + } + cbm_log_warn("index.worker.publication_refused", "reason", + claim == INDEX_PUBLICATION_TAKEN ? "resource_limit" : "claim_failed", "fence", + g_worker_publish_fence); + return false; +} + /* Worker log startup header — see index_supervisor.h. */ static atomic_flag g_worker_log_begun = ATOMIC_FLAG_INIT; @@ -250,6 +307,13 @@ cbm_index_worker_argv_status_t cbm_index_worker_parse_process_argv( parsed.quarantine_file = argv[next + 1]; next += 2; } + if (next < argc && argv[next] && strcmp(argv[next], CBM_INDEX_WORKER_PUBLISH_FENCE_ARG) == 0) { + if (next + 1 >= argc || !argv[next + 1] || !argv[next + 1][0]) { + return CBM_INDEX_WORKER_ARGV_INVALID; + } + parsed.publish_fence_file = argv[next + 1]; + next += 2; + } if (next != argc) { return CBM_INDEX_WORKER_ARGV_INVALID; } @@ -477,12 +541,68 @@ static void worker_log_last_phase(const char *log_path, char *out, size_t out_si enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, - INDEX_WORKER_ARGV_CAP = 17, + INDEX_WORKER_ARGV_CAP = 19, INDEX_WORKER_SYNC_POLL_NS = 10000000, INDEX_WORKER_RELAY_LINES_PER_POLL = 64, INDEX_WORKER_RELAY_BYTES_PER_POLL = 64 * 1024, + INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT = 3, + INDEX_WORKER_RSS_PROBE_INTERVAL_MS = 250, }; +typedef enum { + INDEX_WORKER_TERMINATION_NONE = 0, + INDEX_WORKER_TERMINATION_CANCEL_PENDING, + INDEX_WORKER_TERMINATION_CANCEL, + INDEX_WORKER_TERMINATION_RESOURCE, +} index_worker_termination_reason_t; + +/* Who owns the publication fence of one attempt. Only the polling owner thread + * reads or writes it. */ +typedef enum { + INDEX_WORKER_PUBLICATION_OPEN = 0, /* unclaimed, or no fence at all */ + INDEX_WORKER_PUBLICATION_SUPERVISOR, + INDEX_WORKER_PUBLICATION_WORKER, +} index_worker_publication_owner_t; + +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_index_supervisor_clock_fn g_resource_clock; +static cbm_index_supervisor_rss_fn g_resource_rss; +static void *g_resource_hook_context; + +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context) { + g_resource_clock = clock_fn; + g_resource_rss = rss_fn; + g_resource_hook_context = context; +} + +void cbm_index_supervisor_reset_resource_hooks_for_testing(void) { + g_resource_clock = NULL; + g_resource_rss = NULL; + g_resource_hook_context = NULL; +} +#endif + +static uint64_t worker_resource_now_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_clock) { + return g_resource_clock(g_resource_hook_context); + } +#endif + return cbm_now_ms(); +} + +static cbm_proc_tree_rss_status_t worker_resource_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_resource_rss) { + return g_resource_rss(process, rss_bytes, g_resource_hook_context); + } +#endif + return cbm_subprocess_tree_rss_bytes(process, rss_bytes); +} + struct cbm_index_worker_handle { cbm_subprocess_t *process; char response_path[INDEX_WORKER_PATH_CAP]; @@ -492,10 +612,29 @@ struct cbm_index_worker_handle { long relay_tail_pos; bool process_terminal; cbm_proc_result_t process_result; + cbm_index_resource_policy_t resource_policy; + uint64_t started_ms; + uint64_t last_rss_probe_ms; + unsigned int rss_probe_failures; + bool rss_probe_started; + /* Publication fence: a never-created path next to a reserved unique file + * (both removed at terminal). Empty when no worker limit is enabled. */ + char publish_fence_reserve[INDEX_WORKER_PATH_CAP]; + char publish_fence_path[INDEX_WORKER_PATH_CAP]; + index_worker_publication_owner_t publication_owner; + bool publication_claim_error_logged; + atomic_int termination_reason; atomic_bool terminal; cbm_index_worker_result_t result; }; +#ifdef CBM_ENABLE_TEST_SEAMS +const char *cbm_index_worker_publish_fence_path_for_testing( + const cbm_index_worker_handle_t *handle) { + return handle && handle->publish_fence_path[0] ? handle->publish_fence_path : NULL; +} +#endif + /* The generic subprocess supervisor tails logs in bounded batches to preserve * its nonblocking poll contract. A short-lived worker can therefore become * terminal with more completed lines still on disk. Keep the request callback @@ -670,7 +809,8 @@ static bool worker_unique_file(char *out, size_t out_size, const char *kind) { static bool worker_result_succeeded(const cbm_index_worker_result_t *result) { return result && result->outcome == CBM_PROC_CLEAN && !result->cancellation_requested && - result->tree_quiesced && !result->supervision_failed; + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced && !result->supervision_failed; } static void worker_terminal_log(cbm_index_worker_handle_t *handle) { @@ -701,6 +841,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.supervision_failed || !handle->result.tree_quiesced) { cbm_log_error("index.supervisor.containment_failed", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.resource_probe_failed) { + cbm_log_error("index.supervisor.resource_probe_failed", "resource", "rss_bytes", "log", + handle->log_path); + } else if (handle->result.resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + cbm_log_warn("index.supervisor.resource_limit", "resource", + cbm_index_resource_name(handle->result.resource_violation.resource), "log", + handle->log_path); } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); @@ -742,10 +889,208 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes) { return memory_budget_bytes > SIZE_MAX - headroom ? SIZE_MAX : memory_budget_bytes + headroom; } -int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, - bool single_thread, const char *marker_file, - const char *quarantine_file, cbm_proc_log_cb log_callback, - void *log_context, cbm_index_worker_handle_t **handle_out) { +/* Claim the publication fence before stopping the worker for a resource limit + * (see index_supervisor.h). True when the termination may proceed. */ +static bool worker_claim_publication_for_limit(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + if (!handle->publish_fence_path[0] || + handle->publication_owner == INDEX_WORKER_PUBLICATION_SUPERVISOR) { + return true; + } + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER) { + return false; + } + index_publication_claim_t claim = index_publication_claim(handle->publish_fence_path); + if (claim == INDEX_PUBLICATION_CLAIMED) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_SUPERVISOR; + return true; + } + if (claim == INDEX_PUBLICATION_TAKEN) { + /* The worker committed to publishing before the limit was observed. + * Stopping it now could leave a new generation serving behind a report + * that the previous index was preserved, so this attempt is no longer + * limited; the crossing is recorded here instead. */ + handle->publication_owner = INDEX_WORKER_PUBLICATION_WORKER; + char observed_text[CBM_SZ_32]; + char limit_text[CBM_SZ_32]; + (void)snprintf(observed_text, sizeof(observed_text), "%llu", (unsigned long long)observed); + (void)snprintf(limit_text, sizeof(limit_text), "%llu", (unsigned long long)limit); + cbm_log_warn("index.supervisor.resource_limit_after_publication", "resource", + cbm_index_resource_name(resource), "observed", + probe_failed ? "unmeasured" : observed_text, "limit", limit_text, "action", + "not_enforced", "log", handle->log_path); + return false; + } + /* Neither side holds the fence and it could not be created. Terminating + * now could race a later successful claim by the worker, so the decision + * is retried on the next poll instead. */ + if (!handle->publication_claim_error_logged) { + handle->publication_claim_error_logged = true; + char errno_text[CBM_SZ_32]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", errno); + cbm_log_error("index.supervisor.publication_claim_failed", "errno", errno_text, "action", + "retry_next_poll", "log", handle->log_path); + } + return false; +} + +static void worker_request_resource_termination(cbm_index_worker_handle_t *handle, + cbm_index_resource_t resource, uint64_t observed, + uint64_t limit, bool probe_failed) { + if (!worker_claim_publication_for_limit(handle, resource, observed, limit, probe_failed)) { + return; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + if (!atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_RESOURCE, + memory_order_acq_rel, memory_order_acquire)) { + return; + } + handle->result.resource_violation = (cbm_index_resource_violation_t){ + .resource = resource, .observed = observed, .limit = limit}; + handle->result.resource_probe_failed = probe_failed; + if (!cbm_subprocess_request_cancel(handle->process)) { + handle->result.resource_violation = (cbm_index_resource_violation_t){0}; + handle->result.resource_probe_failed = false; + expected = INDEX_WORKER_TERMINATION_RESOURCE; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + } +} + +static bool worker_check_resource_limits(cbm_index_worker_handle_t *handle) { + /* A worker that committed to publishing is no longer resource-limited. */ + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER || + atomic_load_explicit(&handle->termination_reason, memory_order_acquire) != + INDEX_WORKER_TERMINATION_NONE || + cbm_subprocess_termination_pending(handle->process) || + !cbm_subprocess_supervision_active(handle->process)) { + return false; + } + uint64_t now = 0; + bool have_now = false; + bool rss_probe_failed = false; + if (handle->resource_policy.max_rss_bytes.enabled) { + now = worker_resource_now_ms(); + have_now = true; + bool probe_due = !handle->rss_probe_started || now < handle->last_rss_probe_ms || + now - handle->last_rss_probe_ms >= INDEX_WORKER_RSS_PROBE_INTERVAL_MS; + if (!probe_due) { + goto duration_check; + } + handle->rss_probe_started = true; + handle->last_rss_probe_ms = now; + uint64_t rss_bytes = 0; + cbm_proc_tree_rss_status_t status = worker_resource_rss(handle->process, &rss_bytes); + if (status == CBM_PROC_TREE_RSS_OK) { + handle->rss_probe_failures = 0; + if (rss_bytes > handle->resource_policy.max_rss_bytes.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, rss_bytes, + handle->resource_policy.max_rss_bytes.value, + false); + return false; + } + } else if (status == CBM_PROC_TREE_RSS_ERROR) { + handle->rss_probe_failures++; + if (handle->rss_probe_failures >= INDEX_WORKER_RSS_PROBE_FAILURE_LIMIT) { + rss_probe_failed = true; + } + } else { + handle->rss_probe_failures = 0; + } + } +duration_check: + if (handle->resource_policy.max_duration_ms.enabled) { + if (!have_now) { + now = worker_resource_now_ms(); + } + uint64_t elapsed = now >= handle->started_ms ? now - handle->started_ms : 0; + if (elapsed > handle->resource_policy.max_duration_ms.value) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_DURATION_MS, elapsed, + handle->resource_policy.max_duration_ms.value, + false); + } + } + return rss_probe_failed; +} + +/* Reserve a unique private name and derive the fence path from it. The fence + * itself is never created here: creating it IS the claim. */ +static bool worker_publish_fence_create(cbm_index_worker_handle_t *handle) { + if (!worker_unique_file(handle->publish_fence_reserve, sizeof(handle->publish_fence_reserve), + "publish")) { + return false; + } + int written = snprintf(handle->publish_fence_path, sizeof(handle->publish_fence_path), + "%s.claim", handle->publish_fence_reserve); + if (written <= 0 || written >= (int)sizeof(handle->publish_fence_path)) { + (void)cbm_unlink(handle->publish_fence_reserve); + handle->publish_fence_reserve[0] = '\0'; + handle->publish_fence_path[0] = '\0'; + return false; + } + return true; +} + +static void worker_publish_fence_remove(cbm_index_worker_handle_t *handle) { + if (handle->publish_fence_path[0]) { + (void)cbm_unlink(handle->publish_fence_path); + } + if (handle->publish_fence_reserve[0]) { + (void)cbm_unlink(handle->publish_fence_reserve); + } +} + +/* Create the private files of one attempt. NULL on success; otherwise the name + * of the artifact that could not be created, with errno as that step left it + * and every earlier artifact still in place for the caller to remove. */ +static const char *worker_artifacts_create(cbm_index_worker_handle_t *handle) { + if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response")) { + return "response"; + } + if (!worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) { + return "log"; + } + /* Opt-in: only an enabled worker limit can stop a worker, so only then is + * there a publication to coordinate. */ + if (cbm_index_policy_worker_enabled(&handle->resource_policy) && + !worker_publish_fence_create(handle)) { + return "publish_fence"; + } + return NULL; +} + +/* Settle who held the fence once the attempt is terminal. Claiming an unowned + * fence now also bars any later claim, so a stray member of an unquiesced tree + * can never publish after this report. */ +static bool worker_publication_committed_at_terminal(cbm_index_worker_handle_t *handle) { + if (!handle->publish_fence_path[0] || + handle->publication_owner == INDEX_WORKER_PUBLICATION_SUPERVISOR) { + return false; + } + if (handle->publication_owner == INDEX_WORKER_PUBLICATION_WORKER) { + return true; + } + index_publication_claim_t claim = index_publication_claim(handle->publish_fence_path); + if (claim == INDEX_PUBLICATION_CLAIMED) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_SUPERVISOR; + return false; + } + if (claim == INDEX_PUBLICATION_TAKEN) { + handle->publication_owner = INDEX_WORKER_PUBLICATION_WORKER; + } + /* Taken: the worker committed. Unreadable: preservation cannot be proven. */ + return true; +} + +static int worker_start_internal(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { if (handle_out) { *handle_out = NULL; } @@ -777,17 +1122,22 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (!handle) { return -1; } + cbm_index_policy_init(&handle->resource_policy); + if (resource_policy) { + handle->resource_policy = *resource_policy; + } + atomic_init(&handle->termination_reason, INDEX_WORKER_TERMINATION_NONE); atomic_init(&handle->terminal, false); handle->log_callback = log_callback; handle->log_context = log_context; worker_result_init(&handle->result); - if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response") || - !worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) { + const char *failed_artifact = worker_artifacts_create(handle); + if (failed_artifact) { int saved_errno = errno; char error_text[CBM_SZ_32]; (void)snprintf(error_text, sizeof(error_text), "%d", saved_errno); - cbm_log_error("index.supervisor.artifact_create_failed", "artifact", - handle->response_path[0] ? "log" : "response", "errno", error_text); + cbm_log_error("index.supervisor.artifact_create_failed", "artifact", failed_artifact, + "errno", error_text); (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); free(handle); @@ -822,6 +1172,10 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ argv[argc++] = CBM_INDEX_WORKER_QUARANTINE_ARG; argv[argc++] = quarantine_file; } + if (handle->publish_fence_path[0]) { + argv[argc++] = CBM_INDEX_WORKER_PUBLISH_FENCE_ARG; + argv[argc++] = handle->publish_fence_path; + } argv[argc] = NULL; cbm_proc_opts_t options = {0}; @@ -839,19 +1193,60 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ if (cbm_subprocess_spawn(&options, &handle->process) != 0) { (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); + worker_publish_fence_remove(handle); free(handle); cbm_log_error("index.supervisor.spawn_failed", "action", "fail_closed"); return -1; } + if (handle->resource_policy.max_duration_ms.enabled) { + /* Duration is per request, not per spawn: a crash/hang recovery must + * not reset the clock. Callers pass the request origin; 0 means now. */ + uint64_t origin = duration_origin_ms != 0 ? duration_origin_ms : worker_resource_now_ms(); + handle->started_ms = origin; + } *handle_out = handle; return 0; } +uint64_t cbm_index_worker_now_ms(void) { + return worker_resource_now_ms(); +} + +int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_bytes, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, log_callback, log_context, 0, handle_out); +} + int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out) { - return cbm_index_worker_start_with_log(args_json, memory_budget_bytes, single_thread, - marker_file, quarantine_file, NULL, NULL, handle_out); + return worker_start_internal(args_json, memory_budget_bytes, NULL, single_thread, marker_file, + quarantine_file, NULL, NULL, 0, handle_out); +} + +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, NULL, NULL, duration_origin_ms, + handle_out); +} + +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out) { + return worker_start_internal(args_json, memory_budget_bytes, resource_policy, single_thread, + marker_file, quarantine_file, log_callback, log_context, + duration_origin_ms, handle_out); } cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, @@ -868,10 +1263,16 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool relay_caught_up = true; if (!handle->process_terminal) { + bool rss_probe_failed = worker_check_resource_limits(handle); cbm_proc_result_t process_result; cbm_proc_poll_t state = cbm_subprocess_poll(handle->process, &process_result); relay_caught_up = worker_relay_log(handle); if (state == CBM_PROC_POLL_RUNNING) { + if (rss_probe_failed && cbm_subprocess_root_running(handle->process)) { + worker_request_resource_termination(handle, CBM_INDEX_RESOURCE_RSS_BYTES, 0, + handle->resource_policy.max_rss_bytes.value, + true); + } return CBM_INDEX_WORKER_POLL_RUNNING; } if (state != CBM_PROC_POLL_TERMINAL) { @@ -892,10 +1293,21 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.outcome = process_result->outcome; handle->result.exit_code = process_result->exit_code; handle->result.term_signal = process_result->term_signal; - handle->result.cancellation_requested = process_result->cancellation_requested; + int termination_reason = + atomic_load_explicit(&handle->termination_reason, memory_order_acquire); + handle->result.cancellation_requested = + termination_reason == INDEX_WORKER_TERMINATION_CANCEL || + (termination_reason == INDEX_WORKER_TERMINATION_CANCEL_PENDING && + process_result->cancellation_requested); handle->result.forced = process_result->forced; handle->result.tree_quiesced = process_result->tree_quiesced; handle->result.supervision_failed = process_result->supervision_failed; + handle->result.publication_committed = worker_publication_committed_at_terminal(handle); + if (handle->result.tree_quiesced) { + /* An unquiesced tree keeps the fence as it is: removing a claim the + * supervisor holds would let a surviving member publish later. */ + worker_publish_fence_remove(handle); + } if (worker_result_succeeded(&handle->result)) { worker_response_read_status_t response_status; handle->result.response = slurp_worker_response(handle->response_path, &response_status); @@ -926,8 +1338,36 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, } bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { - return handle && !atomic_load_explicit(&handle->terminal, memory_order_acquire) && - cbm_subprocess_request_cancel(handle->process); + if (!handle || atomic_load_explicit(&handle->terminal, memory_order_acquire)) { + return false; + } + int expected = INDEX_WORKER_TERMINATION_NONE; + bool claimed = atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL_PENDING, + memory_order_acq_rel, memory_order_acquire); + if (!claimed) { + if (expected == INDEX_WORKER_TERMINATION_CANCEL) { + return true; + } + if (expected != INDEX_WORKER_TERMINATION_CANCEL_PENDING) { + return false; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + if (accepted) { + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit( + &handle->termination_reason, &expected, INDEX_WORKER_TERMINATION_CANCEL, + memory_order_acq_rel, memory_order_acquire); + } + return accepted; + } + bool accepted = cbm_subprocess_request_cancel(handle->process); + expected = INDEX_WORKER_TERMINATION_CANCEL_PENDING; + (void)atomic_compare_exchange_strong_explicit(&handle->termination_reason, &expected, + accepted ? INDEX_WORKER_TERMINATION_CANCEL + : INDEX_WORKER_TERMINATION_NONE, + memory_order_acq_rel, memory_order_acquire); + return accepted; } const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { @@ -952,18 +1392,20 @@ void cbm_index_worker_destroy(cbm_index_worker_handle_t *handle) { free(handle); } -int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, - const char *marker_file, const char *quarantine_file, - cbm_proc_log_cb log_callback, void *log_context, - const atomic_int *cancel_requested, - cbm_index_worker_result_t *result) { +static int worker_spawn_internal(const char *args_json, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, + uint64_t duration_origin_ms, cbm_index_worker_result_t *result) { if (!result) { return -1; } worker_result_init(result); cbm_index_worker_handle_t *handle = NULL; - if (cbm_index_worker_start_with_log(args_json, 0, single_thread, marker_file, quarantine_file, - log_callback, log_context, &handle) != 0) { + if (worker_start_internal(args_json, 0, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, duration_origin_ms, + &handle) != 0) { return -1; } const cbm_index_worker_result_t *cached = NULL; @@ -989,6 +1431,25 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th return 0; } +int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_thread, + const char *marker_file, const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + const atomic_int *cancel_requested, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, NULL, single_thread, marker_file, quarantine_file, + log_callback, log_context, cancel_requested, 0, result); +} + +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result) { + return worker_spawn_internal(args_json, resource_policy, single_thread, marker_file, + quarantine_file, log_callback, log_context, cancel_requested, + duration_origin_ms, result); +} + int cbm_index_spawn_worker_with_log(const char *args_json, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, void *log_context, @@ -1020,6 +1481,9 @@ cbm_index_supervised_result_disposition_t cbm_index_supervised_result_dispositio worker_result->supervision_failed) { return CBM_INDEX_SUPERVISED_RESULT_UNSAFE_TERMINAL; } + if (worker_result->resource_violation.resource != CBM_INDEX_RESOURCE_NONE) { + return CBM_INDEX_SUPERVISED_RESULT_RESOURCE_FAILURE; + } if (worker_result->outcome == CBM_PROC_CLEAN) { return worker_result->response ? CBM_INDEX_SUPERVISED_RESULT_SUCCESS : CBM_INDEX_SUPERVISED_RESULT_FALLBACK; diff --git a/src/operations/index_supervisor.h b/src/operations/index_supervisor.h index c023136c0..113653b3e 100644 --- a/src/operations/index_supervisor.h +++ b/src/operations/index_supervisor.h @@ -22,8 +22,10 @@ #include #include +#include #include +#include "foundation/index_policy.h" #include "foundation/subprocess.h" /* cbm_proc_outcome_t */ /* Worker-role state, set once from the CLI arg parser (main.c) when this process @@ -40,6 +42,7 @@ void cbm_index_set_worker_role(bool is_worker, const char *response_out); #define CBM_INDEX_WORKER_QUARANTINE_ARG "--index-worker-quarantine" #define CBM_INDEX_WORKER_MEMORY_BUDGET_ARG "--index-worker-memory-budget-bytes" #define CBM_INDEX_WORKER_BUILD_ARG "--index-worker-build" +#define CBM_INDEX_WORKER_PUBLISH_FENCE_ARG "--index-worker-publish-fence" #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_LENGTH 64U #define CBM_INDEX_WORKER_BUILD_FINGERPRINT_SIZE 65U void cbm_index_set_worker_role_options(bool is_worker, const char *response_out, bool single_thread, @@ -49,6 +52,32 @@ bool cbm_index_worker_active(void); const char *cbm_index_worker_response_out(void); size_t cbm_index_worker_memory_budget_bytes(void); +/* Publication fence. A worker resource limit (index_max_rss_mb, + * index_max_duration_seconds) is enforced by the supervisor, while only the + * worker replaces the serving database. When such a limit is enabled, every + * worker attempt gets a fence file path, and publication and resource + * termination both CLAIM it by creating that file exclusively, so exactly one + * of them can win: + * - the worker claims immediately before it replaces the serving database; + * losing means the supervisor already decided to stop it for a limit, so + * the run aborts before publication and the previous index stays intact; + * - the supervisor claims before it terminates the worker for a limit; + * losing means the worker already committed to publishing, so the limit is + * no longer enforced for that attempt and the attempt runs to completion. + * A reported resource failure therefore never follows a publication by the + * same attempt. Without an enabled worker limit there is no fence and nothing + * changes. The supervisor passes the path on the worker argv; the worker entry + * installs it here (NULL clears it). */ +void cbm_index_set_worker_publish_fence(const char *fence_file); +const char *cbm_index_worker_publish_fence(void); + +/* Worker side: claim publication. True when publishing may proceed: no fence + * is installed, or this call created the fence file. False when the + * supervisor already claimed it (a resource limit stopped this attempt) or the + * claim could not be made; the caller must then leave the serving database + * untouched. Idempotent after a successful claim. */ +bool cbm_index_worker_claim_publication(void); + /* Event name of the worker log's startup header. Shared with the tests so the * contract has exactly one spelling. */ #define CBM_INDEX_WORKER_LOG_START_EVENT "index.worker.start" @@ -94,6 +123,7 @@ typedef struct { const char *marker_file; const char *quarantine_file; size_t memory_budget_bytes; + const char *publish_fence_file; } cbm_index_worker_invocation_t; typedef enum { @@ -142,6 +172,7 @@ int cbm_index_supervisor_spawn_st_count(void); typedef enum { CBM_INDEX_SUPERVISED_RESULT_FALLBACK = 0, CBM_INDEX_SUPERVISED_RESULT_SUCCESS, + CBM_INDEX_SUPERVISED_RESULT_RESOURCE_FAILURE, CBM_INDEX_SUPERVISED_RESULT_CONTAINED_FAILURE, CBM_INDEX_SUPERVISED_RESULT_UNSAFE_TERMINAL, } cbm_index_supervised_result_disposition_t; @@ -164,8 +195,16 @@ typedef struct { char last_phase[128]; char worker_log[4096]; bool response_rejected; /* clean worker exceeded the bounded response protocol */ - char *response; /* worker result only after a contained, uncancelled CLEAN exit; - * borrowed for async polls, caller-owned from the sync wrapper */ + cbm_index_resource_violation_t resource_violation; + bool resource_probe_failed; + /* The worker claimed the publication fence: it committed to replacing the + * serving database (whether or not it got that far). Also true when the + * fence state could not be read, because a report may only claim the + * previous index was preserved when it can prove it. Always false without + * a fence (no worker limit enabled) and for an attempt stopped by a limit. */ + bool publication_committed; + char *response; /* worker result only after a contained, uncancelled CLEAN exit; + * borrowed for async polls, caller-owned from the sync wrapper */ } cbm_index_worker_result_t; cbm_index_supervised_result_disposition_t cbm_index_supervised_result_disposition( @@ -190,6 +229,26 @@ size_t cbm_index_worker_job_memory_limit(size_t memory_budget_bytes); int cbm_index_worker_start(const char *args_json, size_t memory_budget_bytes, bool single_thread, const char *marker_file, const char *quarantine_file, cbm_index_worker_handle_t **handle_out); +int cbm_index_worker_start_with_policy(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out); + +/* Policy plus the request-scoped log callback. The daemon default start path + * needs both: a missing policy fails closed, and worker diagnostics still + * reach the daemon journal. duration_origin_ms is the request clock; 0 means now. */ +int cbm_index_worker_start_with_policy_log(const char *args_json, size_t memory_budget_bytes, + const cbm_index_resource_policy_t *resource_policy, + bool single_thread, const char *marker_file, + const char *quarantine_file, + cbm_proc_log_cb log_callback, void *log_context, + uint64_t duration_origin_ms, + cbm_index_worker_handle_t **handle_out); + +/* Supervisor clock used to stamp a request-scoped duration origin. Tests may + * replace it; production is cbm_now_ms(). */ +uint64_t cbm_index_worker_now_ms(void); /* Request-scoped variant used by interactive local CLI calls. The callback is * invoked by the owner thread while it polls the contained worker; log_context @@ -253,7 +312,26 @@ int cbm_index_spawn_worker_with_log_cancel(const char *args_json, bool single_th cbm_proc_log_cb log_callback, void *log_context, const atomic_int *cancel_requested, cbm_index_worker_result_t *result); +int cbm_index_spawn_worker_with_policy_log_cancel( + const char *args_json, const cbm_index_resource_policy_t *resource_policy, bool single_thread, + const char *marker_file, const char *quarantine_file, cbm_proc_log_cb log_callback, + void *log_context, const atomic_int *cancel_requested, uint64_t duration_origin_ms, + cbm_index_worker_result_t *result); void cbm_index_worker_result_free(cbm_index_worker_result_t *result); +#ifdef CBM_ENABLE_TEST_SEAMS +typedef uint64_t (*cbm_index_supervisor_clock_fn)(void *context); +typedef cbm_proc_tree_rss_status_t (*cbm_index_supervisor_rss_fn)(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context); +void cbm_index_supervisor_set_resource_hooks_for_testing(cbm_index_supervisor_clock_fn clock_fn, + cbm_index_supervisor_rss_fn rss_fn, + void *context); +void cbm_index_supervisor_reset_resource_hooks_for_testing(void); +/* Borrowed fence path of an attempt; NULL when it has no fence. */ +const char *cbm_index_worker_publish_fence_path_for_testing( + const cbm_index_worker_handle_t *handle); +#endif + #endif /* CBM_INDEX_SUPERVISOR_H */ diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index a673e7539..c60f92555 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -1274,7 +1274,11 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * to a docs bundle (2026-09-16 probe: 4,207 junk * edges on JetBrains/Exposed). */ lang == CBM_LANG_HTML || lang == CBM_LANG_VUE || - lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO; + lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO || + /* Swift flags only initializer calls `T.init(...)` + * (#2379, extract_calls.c); every other Swift call + * keeps is_method=false and is unaffected. */ + lang == CBM_LANG_SWIFT; /* Bare-call local-binding suppression. A member call has a receiver the * guard above can reason about; a bare `run()` has none, so that guard * cannot see this class at all. Python-only today because the extraction diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 4713befc7..72e2a2e23 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -3162,7 +3162,9 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB lang == CBM_LANG_ARKTS || /* embedded-script hosts — see pass_calls.c */ lang == CBM_LANG_HTML || lang == CBM_LANG_VUE || - lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO; + lang == CBM_LANG_SVELTE || lang == CBM_LANG_ASTRO || + /* Swift initializer calls only — see pass_calls.c */ + lang == CBM_LANG_SWIFT; /* Bare-call local-binding suppression — see the note in pass_calls.c. * This gate MUST stay identical to the one there. */ bool suppress_weak_local_binding = lang == CBM_LANG_PYTHON; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index c3a601a66..3683a729d 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -322,6 +322,9 @@ struct cbm_pipeline { * per pipeline so concurrent test/process activity cannot cross-trigger. */ void (*before_publish_hook)(cbm_pipeline_t *, const char *, void *); void *before_publish_hook_ctx; + /* Production publication gate (cbm_pipeline_set_publish_gate). */ + cbm_pipeline_publish_gate_fn publish_gate; + void *publish_gate_ctx; /* Test-only: sees the complete graph buffer before the dump releases its indexes. */ void (*before_dump_hook)(const cbm_gbuf_t *, void *); void *before_dump_hook_ctx; @@ -932,6 +935,14 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled) { } } +void cbm_pipeline_set_publish_gate(cbm_pipeline_t *p, cbm_pipeline_publish_gate_fn gate, + void *context) { + if (p) { + p->publish_gate = gate; + p->publish_gate_ctx = gate ? context : NULL; + } +} + void cbm_pipeline_set_doc_candidates_enabled(cbm_pipeline_t *p, bool enabled) { if (p) { p->doc_candidates_choice = @@ -1626,7 +1637,8 @@ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p) { } const cbm_index_resource_policy_t *cbm_pipeline_resource_policy(const cbm_pipeline_t *p) { - return p && cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL; + return p && cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy + : NULL; } cbm_index_resource_violation_t *cbm_pipeline_resource_violation(cbm_pipeline_t *p) { @@ -4452,7 +4464,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .ignore_file = NULL, .max_file_size = 0, .resource_policy = - cbm_index_policy_enabled(&p->resource_policy) ? &p->resource_policy : NULL, + cbm_index_policy_discovery_enabled(&p->resource_policy) ? &p->resource_policy : NULL, .resource_violation = &p->resource_violation, }; cbm_file_info_t *files = NULL; @@ -5296,8 +5308,16 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { cbm_userconfig_t *fresh_config = cbm_userconfig_load_with_source(p->repo_path); bool source_matches = cbm_pipeline_project_source_matches(p, fresh_config); cbm_userconfig_free(fresh_config); + bool publish_allowed = source_matches; if (!source_matches) { cbm_log_error("pipeline.err", "phase", "test_declarations_source_changed"); + } else if (p->publish_gate && !p->publish_gate(p->publish_gate_ctx)) { + /* Last point at which the serving database is still untouched: the + * destination preparation below may already move it aside. */ + cbm_log_warn("pipeline.publish_refused", "path", final_path); + publish_allowed = false; + } + if (!publish_allowed) { cleanup_staging_db(staging_path); free(staging_path); free(final_path); diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 0ff067b18..cd22e42ce 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -65,6 +65,17 @@ void cbm_pipeline_set_persistence(cbm_pipeline_t *p, bool enabled); /* Apply a validated discovery resource policy. The value is copied. */ void cbm_pipeline_set_resource_policy(cbm_pipeline_t *p, const cbm_index_resource_policy_t *policy); +/* Publication gate. cbm_pipeline_run calls it once, after the last + * pre-publication check and immediately before the serving database is + * touched (never for a live no-op, which replaces nothing). Returning false + * aborts the run with CBM_PIPELINE_ABORT_PRESERVE_DB and the previous + * generation stays exactly as it was. A supervised index worker uses it to + * claim publication from its supervisor (see the publication fence in + * mcp/index_supervisor.h). NULL removes the gate. */ +typedef bool (*cbm_pipeline_publish_gate_fn)(void *context); +void cbm_pipeline_set_publish_gate(cbm_pipeline_t *p, cbm_pipeline_publish_gate_fn gate, + void *context); + /* Doc -> code candidates (the doc_candidates setting). Never set, the index * mode decides: on for full, off for moderate and fast. The setter makes an * explicit choice that wins in every mode. Off skips scoring documentation diff --git a/tests/test_cli.c b/tests/test_cli.c index af41e2ecf..7065ce049 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -7426,6 +7426,147 @@ TEST(cli_hermes_stable_shell_context_contract) { PASS(); } +/* A fake HOME holding only ~/.hermes/config.yaml, so install and uninstall + * reach exactly the Hermes editors. */ +typedef struct { + char tmpdir[256]; + char config_path[640]; + char binary_path[640]; + char *saved_home; + char *saved_path; + char *saved_hermes; +} cli_hermes_cycle_t; + +static bool cli_hermes_cycle_open(cli_hermes_cycle_t *cycle, const char *initial_config) { + memset(cycle, 0, sizeof(*cycle)); + snprintf(cycle->tmpdir, sizeof(cycle->tmpdir), "/tmp/cli-hermes-cycle-XXXXXX"); + if (!cbm_mkdtemp(cycle->tmpdir)) { + cycle->tmpdir[0] = '\0'; + return false; + } + cycle->saved_home = save_test_env("HOME"); + cycle->saved_path = save_test_env("PATH"); + cycle->saved_hermes = save_test_env("HERMES_HOME"); + cbm_setenv("HOME", cycle->tmpdir, 1); + cbm_setenv("PATH", cycle->tmpdir, 1); + cbm_unsetenv("HERMES_HOME"); + + char hermes_dir[512]; + snprintf(hermes_dir, sizeof(hermes_dir), "%s/.hermes", cycle->tmpdir); + snprintf(cycle->config_path, sizeof(cycle->config_path), "%s/config.yaml", hermes_dir); +#ifdef _WIN32 + snprintf(cycle->binary_path, sizeof(cycle->binary_path), + "%s/.local/bin/codebase-memory-cli.exe", cycle->tmpdir); +#else + snprintf(cycle->binary_path, sizeof(cycle->binary_path), "%s/.local/bin/codebase-memory-cli", + cycle->tmpdir); +#endif + test_mkdirp(hermes_dir); + return write_test_file(cycle->config_path, initial_config) == 0; +} + +static void cli_hermes_cycle_close(cli_hermes_cycle_t *cycle) { + if (!cycle->tmpdir[0]) { + return; + } + restore_test_env("HOME", cycle->saved_home); + restore_test_env("PATH", cycle->saved_path); + restore_test_env("HERMES_HOME", cycle->saved_hermes); + test_rmdir_r(cycle->tmpdir); + cycle->tmpdir[0] = '\0'; +} + +/* Found while finishing #1862: when our hook was the only entry under + * `hooks:`, uninstall empties that mapping and the mapping editor writes it as + * `hooks: {}` (the same convention as `mcp_servers: {}`). The hook sequence + * editor then refused the file, so install -> uninstall -> install exited 1, + * and so did a repeated uninstall. The explicit empty mapping must be accepted: + * a repeated uninstall changes nothing and the reinstall restores the first + * install byte for byte, with exactly one owned hook. */ +TEST(cli_hermes_hook_reinstalls_after_uninstall_empties_hooks) { + cli_hermes_cycle_t cycle; + if (!cli_hermes_cycle_open(&cycle, "theme: solarized\n")) { + cli_hermes_cycle_close(&cycle); + FAIL("Hermes fake-HOME setup failed"); + } + char *argv[] = {"uninstall", "--yes"}; + int install_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); + char *installed = read_test_file_alloc(cycle.config_path); + int uninstall_rc = cli_test_cmd_uninstall(2, argv); + char *emptied = read_test_file_alloc(cycle.config_path); + int repeat_rc = cli_test_cmd_uninstall(2, argv); + char *repeated = read_test_file_alloc(cycle.config_path); + int reinstall_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); + char *reinstalled = read_test_file_alloc(cycle.config_path); + cli_hermes_cycle_close(&cycle); + + bool installed_once = install_rc == 0 && installed && + test_count_substring(installed, "pre_llm_call:") == 1U && + test_count_substring(installed, "hook-augment") == 1U; + bool emptied_clean = uninstall_rc == 0 && emptied && strstr(emptied, "theme: solarized\n") && + test_count_substring(emptied, "hooks: {}\n") == 1U && + !strstr(emptied, "pre_llm_call") && !strstr(emptied, "hook-augment"); + bool repeat_unchanged = repeat_rc == 0 && emptied && repeated && strcmp(repeated, emptied) == 0; + bool reinstalled_once = + reinstall_rc == 0 && installed && reinstalled && strcmp(reinstalled, installed) == 0; + free(installed); + free(emptied); + free(repeated); + free(reinstalled); + if (!installed_once) + FAIL("Hermes install must write exactly one owned pre_llm_call hook"); + if (!emptied_clean) + FAIL("uninstall must remove the owned hook and leave only the empty hooks: {} mapping"); + if (!repeat_unchanged) + FAIL("a repeated uninstall must accept hooks: {} and leave the file unchanged"); + if (!reinstalled_once) + FAIL("reinstall must accept hooks: {} and restore exactly one owned hook"); + PASS(); +} + +/* The same cycle with a user's own hook in the shared pre_llm_call sequence: + * the foreign hook survives untouched and uninstall restores the original + * file byte for byte. */ +TEST(cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook) { + static const char original[] = "theme: solarized\n" + "mcp_servers: {}\n" + "hooks:\n" + " pre_llm_call:\n" + " - command: \"/usr/bin/user-hermes-hook\"\n"; + cli_hermes_cycle_t cycle; + if (!cli_hermes_cycle_open(&cycle, original)) { + cli_hermes_cycle_close(&cycle); + FAIL("Hermes fake-HOME setup failed"); + } + char *argv[] = {"uninstall", "--yes"}; + int install_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); + char *installed = read_test_file_alloc(cycle.config_path); + int uninstall_rc = cli_test_cmd_uninstall(2, argv); + char *restored = read_test_file_alloc(cycle.config_path); + int reinstall_rc = + cbm_install_agent_hooks_for_testing(cycle.tmpdir, cycle.binary_path, false, false); + char *reinstalled = read_test_file_alloc(cycle.config_path); + cli_hermes_cycle_close(&cycle); + + bool merged = install_rc == 0 && installed && + test_count_substring(installed, "/usr/bin/user-hermes-hook") == 1U && + test_count_substring(installed, "pre_llm_call:") == 1U && + test_count_substring(installed, "hook-augment") == 1U; + bool restored_exactly = uninstall_rc == 0 && restored && strcmp(restored, original) == 0; + bool reinstalled_once = + reinstall_rc == 0 && installed && reinstalled && strcmp(reinstalled, installed) == 0; + free(installed); + free(restored); + free(reinstalled); + if (!merged || !restored_exactly || !reinstalled_once) + FAIL("a user's own pre_llm_call hook must survive install, uninstall and reinstall, and " + "uninstall must restore the original file byte for byte"); + PASS(); +} + #ifndef _WIN32 TEST(cli_detected_agent_summary_includes_registry_clients) { char tmpdir[256]; @@ -17626,6 +17767,8 @@ SUITE(cli) { RUN_TEST(cli_junie_current_durable_context_contract); RUN_TEST(cli_rovo_installs_documented_global_memory); RUN_TEST(cli_hermes_stable_shell_context_contract); + RUN_TEST(cli_hermes_hook_reinstalls_after_uninstall_empties_hooks); + RUN_TEST(cli_hermes_hook_cycle_keeps_foreign_pre_llm_hook); #ifndef _WIN32 RUN_TEST(cli_detected_agent_summary_includes_registry_clients); #endif diff --git a/tests/test_config_yaml_edit.c b/tests/test_config_yaml_edit.c index b21bd8b3b..1f48cfe6e 100644 --- a/tests/test_config_yaml_edit.c +++ b/tests/test_config_yaml_edit.c @@ -2075,6 +2075,129 @@ TEST(config_yaml_edit_nested_sequence_removes_only_exact_canonical_item) { PASS(); } +/* Found while finishing #1862: cbm_yaml_remove_mapping_entry writes a section + * it empties as `section: {}`, and the Hermes remover empties `hooks:` whenever + * our hook was its only entry. The sequence editor refused that explicit empty + * parent mapping, so the next install (and a repeated removal) failed. The + * primitives' own round trip must land back on the first insert. */ +TEST(config_yaml_edit_nested_sequence_reinserts_under_emptied_parent) { + const char *initial = "theme: x\n"; + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, initial), 0); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *first = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(first); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + ASSERT_EQ(cbm_yaml_remove_mapping_entry(fixture.path, "hooks", "pre_llm_call"), 0); + char *emptied = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(emptied); + ASSERT_STR_EQ(emptied, "theme: x\nhooks: {}\n"); + free(emptied); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, "id", + yaml_hook_identity, yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *again = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(again); + ASSERT_STR_EQ(again, first); + free(again); + free(first); + th_cleanup(fixture.dir); + PASS(); +} + +/* An exact empty flow mapping on a PARENT of the sequence path is an empty + * mapping, the same `section: {}` the mapping-entry editor writes and accepts. + * Removal is a byte-identical no-op; upsert expands the header in place, + * keeping its comment, the line endings and every other byte. */ +TEST(config_yaml_edit_nested_sequence_expands_explicit_empty_parent_mapping) { + static const char item_lf[] = " pre_llm_call:\n" + " - id: \"codebase-memory-mcp\"\n" + " type: \"command\"\n" + " command: \"/opt/Codebase Memory/bin/cbm\"\n"; + static const char item_crlf[] = " pre_llm_call:\r\n" + " - id: \"codebase-memory-mcp\"\r\n" + " type: \"command\"\r\n" + " command: \"/opt/Codebase Memory/bin/cbm\"\r\n"; + struct { + const char *initial; + const char *header; + const char *item; + const char *tail; + } cases[] = { + {"theme: x\nhooks: {} # keep me\nafter: 1\n", "theme: x\nhooks: # keep me\n", item_lf, + "after: 1\n"}, + {"theme: x\r\nhooks: {}\r\nafter: 1\r\n", "theme: x\r\nhooks:\r\n", item_crlf, + "after: 1\r\n"}, + {"theme: x\nhooks: {}", "theme: x\nhooks:\n", item_lf, ""}, + {"hooks: {}\n# trailing note\n", "hooks:\n# trailing note\n", item_lf, ""}, + }; + for (size_t i = 0U; i < sizeof(cases) / sizeof(cases[0]); i++) { + char expected[1024]; + ASSERT(snprintf(expected, sizeof(expected), "%s%s%s", cases[i].header, cases[i].item, + cases[i].tail) > 0); + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, cases[i].initial), 0); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *unchanged = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(unchanged); + ASSERT_STR_EQ(unchanged, cases[i].initial); + free(unchanged); + for (int pass = 0; pass < 2; pass++) { + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, + 2U, "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_OK); + char *expanded = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(expanded); + ASSERT_STR_EQ(expanded, expected); + free(expanded); + } + th_cleanup(fixture.dir); + } + PASS(); +} + +/* Only an exact `{}` on a parent with nothing nested beneath it is accepted. + * The sequence key itself holding a mapping, a non-empty flow mapping, a flow + * sequence where a mapping belongs, and content indented under an inline `{}` + * stay refused byte-identically by both upsert and removal. */ +TEST(config_yaml_edit_nested_sequence_still_refuses_non_empty_parent_flow) { + const char *cases[] = { + "hooks:\n pre_llm_call: {}\n", + "hooks: {pre_llm_call: []}\n", + "hooks: []\n", + "hooks: {}\n pre_llm_call:\n - id: \"codebase-memory-mcp\"\n", + }; + for (size_t i = 0U; i < sizeof(cases) / sizeof(cases[0]); i++) { + yaml_fixture_t fixture; + ASSERT_EQ(yaml_fixture_init(&fixture, cases[i]), 0); + ASSERT_EQ(cbm_yaml_upsert_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_ERROR); + ASSERT_EQ(cbm_yaml_remove_mapping_sequence_item(fixture.path, yaml_hook_sequence_path, 2U, + "id", yaml_hook_identity, + yaml_hook_canonical_item), + CBM_YAML_IDENTITY_EDIT_ERROR); + char *after = yaml_read_alloc(fixture.path); + ASSERT_NOT_NULL(after); + ASSERT_STR_EQ(after, cases[i]); + free(after); + th_cleanup(fixture.dir); + } + PASS(); +} + /* Uninstall must be idempotent against a config that was never written. Hermes * is detected by its binary on PATH, so a HOME without `.hermes/` reached the * YAML removers with an absent target — and they failed acquiring the lock @@ -2326,6 +2449,9 @@ SUITE(config_yaml_edit) { RUN_TEST(config_yaml_edit_nested_sequence_preserves_crlf); RUN_TEST(config_yaml_edit_nested_sequence_foreign_identity_is_preserved); RUN_TEST(config_yaml_edit_nested_sequence_removes_only_exact_canonical_item); + RUN_TEST(config_yaml_edit_nested_sequence_reinserts_under_emptied_parent); + RUN_TEST(config_yaml_edit_nested_sequence_expands_explicit_empty_parent_mapping); + RUN_TEST(config_yaml_edit_nested_sequence_still_refuses_non_empty_parent_flow); RUN_TEST(config_yaml_edit_nested_sequence_ignores_foreign_flow_and_block_scalars_issue2209); RUN_TEST(config_yaml_edit_nested_sequence_ambiguity_fails_byte_identically); RUN_TEST(config_yaml_edit_remove_on_absent_config_is_a_no_op); diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 44c764434..ce7a07f2a 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1488,6 +1488,8 @@ typedef struct { cbm_project_lock_manager_t *project_locks; bool project_lock_busy_is_error; cbm_proc_outcome_t outcomes[APP_FAKE_MAX_ATTEMPTS]; + cbm_index_resource_violation_t resource_violations[APP_FAKE_MAX_ATTEMPTS]; + bool resource_probe_failures[APP_FAKE_MAX_ATTEMPTS]; const char *responses[APP_FAKE_MAX_ATTEMPTS]; bool response_errors[APP_FAKE_MAX_ATTEMPTS]; const char *marker_payloads[APP_FAKE_MAX_ATTEMPTS]; @@ -1541,7 +1543,9 @@ static void app_fake_worker_read_file(const char *path, char *out, size_t out_si static int app_fake_worker_start(void *opaque, const char *args_json, size_t memory_budget_bytes, const char *marker_file, const char *quarantine_file, + uint64_t duration_origin_ms, cbm_daemon_application_worker_t *worker_out) { + (void)duration_origin_ms; app_fake_worker_context_t *context = opaque; app_fake_worker_t *worker = calloc(1, sizeof(*worker)); if (!worker) { @@ -1596,7 +1600,9 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem static bool app_wait_for_atomic_bool(atomic_bool *value, bool expected); static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, int attempt, - const char *max_files, const char *max_source_mb) { + const char *max_files, const char *max_source_mb, + const char *max_rss_mb, + const char *max_duration_seconds) { if (!context || attempt < 0 || attempt >= APP_FAKE_MAX_ATTEMPTS) { return false; } @@ -1613,9 +1619,18 @@ static bool app_fake_worker_policy_equals(app_fake_worker_context_t *context, in yyjson_val *bytes = policy && yyjson_is_obj(policy) ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB) : NULL; - bool equal = files && bytes && yyjson_is_str(files) && yyjson_is_str(bytes) && + yyjson_val *rss = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_RSS_MB) + : NULL; + yyjson_val *duration = policy && yyjson_is_obj(policy) + ? yyjson_obj_get(policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) + : NULL; + bool equal = files && bytes && rss && duration && yyjson_is_str(files) && + yyjson_is_str(bytes) && yyjson_is_str(rss) && yyjson_is_str(duration) && strcmp(yyjson_get_str(files), max_files) == 0 && - strcmp(yyjson_get_str(bytes), max_source_mb) == 0; + strcmp(yyjson_get_str(bytes), max_source_mb) == 0 && + strcmp(yyjson_get_str(rss), max_rss_mb) == 0 && + strcmp(yyjson_get_str(duration), max_duration_seconds) == 0; yyjson_doc_free(document); return equal; } @@ -1657,6 +1672,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->result.outcome = outcome; worker->result.exit_code = outcome == CBM_PROC_CLEAN ? 0 : -1; worker->result.tree_quiesced = true; + if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + worker->result.resource_violation = + worker->context->resource_violations[worker->attempt]; + worker->result.resource_probe_failed = + worker->context->resource_probe_failures[worker->attempt]; + } const char *response = worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] @@ -2138,11 +2159,13 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool dirs_ok = cbm_mkdtemp(root) != NULL && cbm_mkdtemp(cache) != NULL; bool cache_set = dirs_ok && cache_saved && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; cbm_config_t *stored_config = cache_set ? cbm_config_open(cache) : NULL; - bool config_ready = stored_config && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && - cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0; + bool config_ready = + stored_config && cbm_config_set(stored_config, CBM_CONFIG_AUTO_INDEX, "true") == 0 && + cbm_config_set(stored_config, CBM_CONFIG_AUTO_WATCH, "false") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "3") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "4") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; char canonical_root[APP_TEST_PATH_CAP] = {0}; bool canonical = dirs_ok && cbm_canonical_path(root, canonical_root, sizeof(canonical_root)); char *project = canonical ? cbm_project_name_from_path(canonical_root) : NULL; @@ -2184,7 +2207,8 @@ TEST(daemon_application_initialize_coalesces_auto_index_for_full_sessions) { bool first_owned = first_initialized && project && app_wait_for_subscribers(application, project, 1) && app_wait_for_atomic_int(&fake.starts, 1); - bool auto_policy_propagated = first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4"); + bool auto_policy_propagated = + first_owned && app_fake_worker_policy_equals(&fake, 0, "3", "4", "64", "7"); bool second_initialized = app_test_initialize_session(&callbacks, sessions[1], root, NULL, NULL); bool coalesced = first_owned && second_initialized && project && @@ -2638,9 +2662,11 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { ASSERT_NOT_NULL(root); ASSERT_NOT_NULL(cache); cbm_config_t *stored_config = cbm_config_open(cache); - bool configured = stored_config && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && - cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0; + bool configured = + stored_config && cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_FILES, "5") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "6") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64") == 0 && + cbm_config_set(stored_config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7") == 0; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); @@ -2660,7 +2686,8 @@ TEST(daemon_application_programmatic_index_injects_resource_policy) { cbm_daemon_application_t *application = configured ? cbm_daemon_application_new(&config) : NULL; int index_rc = application ? cbm_daemon_application_index(application, "policy-programmatic", root) : -1; - bool policy_propagated = index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6"); + bool policy_propagated = + index_rc == 0 && app_fake_worker_policy_equals(&fake, 0, "5", "6", "64", "7"); bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); cbm_daemon_application_free(application); @@ -2746,6 +2773,188 @@ TEST(daemon_application_programmatic_index_forwards_set_doc_candidates) { PASS(); } +TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried) { + /* th_mktempdir returns one static buffer: copy the cache path. */ + char *cache_made = th_mktempdir("cbm_app_worker_resource_cache"); + char *cache = cache_made ? strdup(cache_made) : NULL; + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_KILLED; + fake.resource_violations[0] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4188); + char *root_made = th_mktempdir("cbm_app_worker_resource"); + char *root = root_made ? strdup(root_made) : NULL; + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + /* The CLI daemon answers with the native result frame, not an MCP envelope: + * the payload JSON is unescaped and the frame is not NUL-terminated. */ + bool structured = + app_test_response_contains(response, response_length, "resource_limit_exceeded") && + app_test_response_contains(response, response_length, "\"stage\":\"worker\"") && + app_test_response_contains(response, response_length, "\"resource\":\"duration_ms\"") && + app_test_response_contains(response, response_length, "\"observed\":7001") && + app_test_response_contains(response, response_length, "\"limit\":7000") && + app_test_response_contains(response, response_length, "\"unit\":\"milliseconds\"") && + response && response_length > 1 && response[0] == 1; + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + free(root); + free(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 1); + ASSERT_EQ(destroys, 1); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(daemon_application_worker_resource_failure_after_crash_stops_recovery) { + /* th_mktempdir returns one static buffer: copy the cache path. */ + char *cache_made = th_mktempdir("cbm_app_worker_resource_crash_cache"); + char *cache = cache_made ? strdup(cache_made) : NULL; + cbm_config_t *stored_config = cache ? cbm_config_open(cache) : NULL; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + /* A crash starts recovery; the recovery attempt that hits a limit ends it. */ + fake.outcomes[0] = CBM_PROC_CRASH; + fake.outcomes[1] = CBM_PROC_CRASH; + fake.marker_payloads[1] = "S crashing.c\n"; + fake.outcomes[2] = CBM_PROC_KILLED; + fake.resource_violations[2] = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 7001, .limit = 7000}; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .config = stored_config, + .worker_ops = &worker_ops, + }; + cbm_daemon_application_t *application = + stored_config ? cbm_daemon_application_new(&config) : NULL; + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 4189); + char *root_made = th_mktempdir("cbm_app_worker_resource_crash"); + char *root = root_made ? strdup(root_made) : NULL; + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 96]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"name\":\"DaemonWorkerResourceCrashFixture\"}", + root ? root : ""); + bool setup = cache && stored_config && application && session && root && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + if (setup) { + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + setup = app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + } + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR; + /* The CLI daemon answers with the native result frame, not an MCP envelope: + * the payload JSON is unescaped and the frame is not NUL-terminated. */ + bool structured = + app_test_response_contains(response, response_length, "resource_limit_exceeded") && + app_test_response_contains(response, response_length, "\"stage\":\"worker\"") && + app_test_response_contains(response, response_length, "\"resource\":\"duration_ms\"") && + app_test_response_contains(response, response_length, "\"observed\":7001") && + app_test_response_contains(response, response_length, "\"limit\":7000") && + app_test_response_contains(response, response_length, "\"unit\":\"milliseconds\"") && + response && response_length > 1 && response[0] == 1; + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + int starts = atomic_load(&fake.starts); + int destroys = atomic_load(&fake.destroys); + cbm_daemon_application_free(application); + cbm_config_close(stored_config); + free(response); + free(context); + free(tool); + th_cleanup(root); + th_cleanup(cache); + free(root); + free(cache); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE(structured); + ASSERT_EQ(starts, 3); + ASSERT_EQ(destroys, 3); + ASSERT_TRUE(stopped); + PASS(); +} + /* #1818: the daemon.autoindex.skipped warning must name the effective limit, * like the MCP autoindex.skip event does (#1466). The warning is emitted on the * background-initialize thread, before the release increment of the @@ -7526,6 +7735,8 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_unnamed_async_and_status_refuse_ambiguous_root_issue2134); RUN_TEST(daemon_application_programmatic_index_injects_resource_policy); RUN_TEST(daemon_application_programmatic_index_forwards_set_doc_candidates); + RUN_TEST(daemon_application_worker_resource_failure_is_structured_and_not_retried); + RUN_TEST(daemon_application_worker_resource_failure_after_crash_stops_recovery); RUN_TEST(daemon_application_auto_index_honors_tracked_file_limit); RUN_TEST(daemon_application_auto_index_file_count_handles_literal_metacharacter_path); RUN_TEST(daemon_application_auto_index_file_count_supports_non_git_roots); diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index a881ffd07..56ded2e13 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -1028,7 +1028,7 @@ static void *runtime_application_client_request_thread(void *opaque) { call->status = call->tagged ? cbm_daemon_runtime_client_application_request_tagged( call->client, call->request_token, call->request, call->request_length, - &call->response, &call->response_length, RUNTIME_TEST_TIMEOUT_MS) + &call->response, &call->response_length, NULL, RUNTIME_TEST_TIMEOUT_MS) : cbm_daemon_runtime_client_application_request( call->client, call->request, call->request_length, &call->response, &call->response_length, RUNTIME_TEST_TIMEOUT_MS); @@ -3853,7 +3853,7 @@ TEST(daemon_runtime_presend_request_cancel_is_sticky_and_nonterminal) { cancel = cbm_daemon_runtime_client_application_cancel(client, request_token); cancelled_status = cbm_daemon_runtime_client_application_request_tagged( client, request_token, blocking_request, (uint32_t)sizeof(blocking_request), - &cancelled_response, &cancelled_response_length, RUNTIME_TEST_TIMEOUT_MS); + &cancelled_response, &cancelled_response_length, NULL, RUNTIME_TEST_TIMEOUT_MS); } if (cancelled_status == CBM_DAEMON_RUNTIME_APPLICATION_CANCELLED) { next_status = cbm_daemon_runtime_client_application_request( @@ -3910,8 +3910,10 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { bool first_reserved = false; bool duplicate_reservation_rejected = false; bool first_exact = false; + bool first_sent = false; bool second_reserved = false; bool second_exact = false; + bool second_sent = false; bool closed = false; bool exited = false; @@ -3931,7 +3933,7 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { cbm_daemon_runtime_application_status_t status = cbm_daemon_runtime_client_application_request_tagged( client, first_token, first_request, (uint32_t)sizeof(first_request), &response, - &response_length, RUNTIME_TEST_TIMEOUT_MS); + &response_length, &first_sent, RUNTIME_TEST_TIMEOUT_MS); first_exact = status == CBM_DAEMON_RUNTIME_APPLICATION_OK && response_length == sizeof(first_request) && response && memcmp(response, first_request, sizeof(first_request)) == 0; @@ -3947,7 +3949,7 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { cbm_daemon_runtime_application_status_t status = cbm_daemon_runtime_client_application_request_tagged( client, second_token, second_request, (uint32_t)sizeof(second_request), &response, - &response_length, RUNTIME_TEST_TIMEOUT_MS); + &response_length, &second_sent, RUNTIME_TEST_TIMEOUT_MS); second_exact = status == CBM_DAEMON_RUNTIME_APPLICATION_OK && response_length == sizeof(second_request) && response && memcmp(response, second_request, sizeof(second_request)) == 0; @@ -3966,9 +3968,11 @@ TEST(daemon_runtime_allows_only_one_unstarted_application_token) { ASSERT_TRUE(first_reserved); ASSERT_TRUE(duplicate_reservation_rejected); ASSERT_TRUE(first_exact); + ASSERT_TRUE(first_sent); ASSERT_TRUE(second_reserved); ASSERT_EQ(second_token, first_token + 1U); ASSERT_TRUE(second_exact); + ASSERT_TRUE(second_sent); ASSERT_TRUE(closed); ASSERT_TRUE(exited); ASSERT_EQ(atomic_load(&context.requests), 2); @@ -4068,6 +4072,10 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { uint8_t *response = NULL; uint32_t response_length = 0; cbm_daemon_runtime_application_status_t status = CBM_DAEMON_RUNTIME_APPLICATION_OK; + cbm_daemon_runtime_application_token_t request_token = + CBM_DAEMON_RUNTIME_APPLICATION_TOKEN_INVALID; + bool token_reserved = false; + bool request_sent = true; bool close_begun = false; bool duplicate_begin_rejected = false; bool close_acknowledged = false; @@ -4078,6 +4086,8 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { RUNTIME_TEST_TIMEOUT_MS, &result); } if (client) { + token_reserved = + cbm_daemon_runtime_client_application_token_reserve(client, &request_token); close_begun = cbm_daemon_runtime_client_close_begin(client); duplicate_begin_rejected = close_begun && !cbm_daemon_runtime_client_close_begin(client); } @@ -4085,9 +4095,9 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { /* Deterministically models the frontend boundary where close begins * after a worker claims an item but before it enters the runtime API. * The retained allocation must reject the call without touching IPC. */ - status = cbm_daemon_runtime_client_application_request( - client, request, (uint32_t)sizeof(request), &response, &response_length, - RUNTIME_TEST_TIMEOUT_MS); + status = cbm_daemon_runtime_client_application_request_tagged( + client, request_token, request, (uint32_t)sizeof(request), &response, &response_length, + &request_sent, RUNTIME_TEST_TIMEOUT_MS); close_acknowledged = cbm_daemon_runtime_client_close_finish(client, RUNTIME_TEST_TIMEOUT_MS); client = NULL; @@ -4105,11 +4115,13 @@ TEST(daemon_runtime_close_begin_retains_storage_and_rejects_late_exchange) { runtime_test_fixture_finish(&fixture); ASSERT_TRUE(started); + ASSERT_TRUE(token_reserved); ASSERT_TRUE(close_begun); ASSERT_TRUE(duplicate_begin_rejected); ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_TRANSPORT_ERROR); ASSERT_NULL(response); ASSERT_EQ(response_length, 0); + ASSERT_FALSE(request_sent); ASSERT_TRUE(close_acknowledged); ASSERT_TRUE(exited); ASSERT_EQ(atomic_load(&context.opened), 1); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 1d9638b22..215800a0a 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -83,6 +83,28 @@ static int has_def_qn(CBMFileResult *r, const char *qn) { return 0; } +/* Find a definition by its qualified name relative to the file's module QN + * (e.g. "Box.init" resolves to ".Box.init"). */ +static const CBMDefinition *find_def_in_module(CBMFileResult *r, const char *rel_qn) { + char qn[512]; + snprintf(qn, sizeof(qn), "%s.%s", r->module_qn, rel_qn); + for (int i = 0; i < r->defs.count; i++) { + if (r->defs.items[i].qualified_name && strcmp(r->defs.items[i].qualified_name, qn) == 0) + return &r->defs.items[i]; + } + return NULL; +} + +/* Find an in-body call by its raw callee text; returns the call or NULL. */ +static const CBMCall *find_call_by_callee(CBMFileResult *r, const char *callee) { + for (int i = 0; i < r->calls.count; i++) { + if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) { + return &r->calls.items[i]; + } + } + return NULL; +} + static int count_defs_with_label(CBMFileResult *r, const char *label) { int count = 0; for (int i = 0; i < r->defs.count; i++) { @@ -902,6 +924,223 @@ TEST(php_function) { PASS(); } +/* #2000: inline HTML between `?>` and `` fell into an + * ERROR region and never reached the graph. */ +static const CBMCall *find_call_named(CBMFileResult *r, const char *callee) { + for (int i = 0; i < r->calls.count; i++) { + if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) + return &r->calls.items[i]; + } + return NULL; +} + +static int php_def_line(CBMFileResult *r, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + if (strcmp(r->defs.items[i].name, name) == 0) + return (int)r->defs.items[i].start_line; + } + return -1; +} + +TEST(php_inline_html_tail_reaches_graph_issue2000) { + CBMFileResult *r = extract("\n" + "
\n" + "

plain markup after the closing tag

\n" + "
\n" + "has_error); + ASSERT(has_def(r, "Function", "gamma")); + ASSERT(has_def(r, "Function", "delta")); + /* Lines are in original-file coordinates. */ + ASSERT_EQ(php_def_line(r, "gamma"), 2); + ASSERT_EQ(php_def_line(r, "delta"), 8); + const CBMCall *c = find_call_named(r, "gamma"); + ASSERT_NOT_NULL(c); + ASSERT_NOT_NULL(strstr(c->enclosing_func_qn, "delta")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* View-file shapes: echo tags keep their calls, and control structures that + * are opened in one PHP block and closed in a later one still parse. */ +TEST(php_inline_html_template_shapes_issue2000) { + CBMFileResult *r = extract("\n" + "<?= page_title($page) ?>\n" + "\n" + "\n" + "

Hello name?>

\n" + "\n" + " log in\n" + "\n" + "\n" + "
    • \n" + "\n" + "\n" + "\n", + CBM_LANG_PHP, "t", "view.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_call(r, "page_title")); + ASSERT(has_call(r, "render_item")); + ASSERT(has_def(r, "Function", "footer_links")); + ASSERT_EQ(php_def_line(r, "footer_links"), 12); + ASSERT(has_call(r, "nav_links")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* `?>` only leaves PHP mode from code or a one-line comment. Inside strings, + * heredoc, nowdoc and block comments it is text; `#[` is an attribute, not a + * comment; ` quoted';\n" + "$b = \"double ?> quoted \\\" still\";\n" + "$c = << body\n" + " EOT;\n" + "$d = <<<'NOW'\n" + "nowdoc ?> body\n" + "NOW;\n" + "/* block ?> comment */\n" + "#[Pure]\n" + "function one() { return 1; }\n" + "// line comment ends here ?>\n" + "\n" + "has_error); + ASSERT(has_def(r, "Function", "one")); + ASSERT(has_def(r, "Function", "two")); + ASSERT_EQ(php_def_line(r, "two"), 16); + ASSERT(has_call(r, "one")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* Markup is blanked, never parsed: code-shaped text outside the PHP tags must + * not produce definitions or calls, wherever it sits. */ +TEST(php_inline_html_markup_never_reaches_graph_issue2000) { + CBMFileResult *r = extract("

    function before() { return leakBefore(); }

    \n" + "\n" + "

    function fake() { return leak(); }

    \n" + "\n" + "function xmlFake() { leakXml(); }\n" + "\n" + "

    class Tail { function tailFake() { leakTail(); } }

    \n", + CBM_LANG_PHP, "t", "markup.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Function", "real")); + ASSERT(has_def(r, "Function", "after")); + ASSERT(has_call(r, "real")); + ASSERT_FALSE(has_def_any(r, "before")); + ASSERT_FALSE(has_def_any(r, "fake")); + ASSERT_FALSE(has_def_any(r, "jsFake")); + ASSERT_FALSE(has_def_any(r, "xmlFake")); + ASSERT_FALSE(has_def_any(r, "Tail")); + ASSERT_FALSE(has_def_any(r, "tailFake")); + ASSERT_FALSE(has_call(r, "leak")); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* The mask hands back the caller's buffer, uncopied, when nothing needs + * rewriting, and a same-length rewrite otherwise. The pointer alone cannot + * tell copy-on-write from an eager copy, so the arena's allocation total is + * sampled around each call: no bytes for `plain` and `quoted`, a copy for + * `mixed`. */ +TEST(php_inline_html_mask_copies_only_on_write_issue2000) { + CBMArena arena; + cbm_arena_init(&arena); + const char *plain = "';\n/* ?> */\n$h = <<\nEOT;\n"; + const char *mixed = "\nx\n"; + + size_t before = cbm_arena_total(&arena); + ASSERT(cbm_php_mask_inline_html(&arena, plain, (int)strlen(plain)) == plain); + ASSERT_EQ(cbm_arena_total(&arena), before); + + before = cbm_arena_total(&arena); + ASSERT(cbm_php_mask_inline_html(&arena, quoted, (int)strlen(quoted)) == quoted); + ASSERT_EQ(cbm_arena_total(&arena), before); + + before = cbm_arena_total(&arena); + const char *masked = cbm_php_mask_inline_html(&arena, mixed, (int)strlen(mixed)); + ASSERT(cbm_arena_total(&arena) > before); + ASSERT(masked != mixed); + ASSERT_EQ((int)strlen(masked), (int)strlen(mixed)); + ASSERT_STR_EQ(masked, "` (common at the end of older class files) is not a partial + * parse, and CRLF line endings keep their line numbers. */ +TEST(php_inline_html_trailing_close_tag_crlf_issue2000) { + CBMFileResult *r = extract("\r\n", + CBM_LANG_PHP, "t", "Repo.php"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + ASSERT(has_def(r, "Class", "Repo")); + ASSERT(has_def(r, "Method", "find")); + ASSERT_EQ(php_def_line(r, "find"), 3); + ASSERT_FALSE(r->parse_incomplete); + cbm_free_result(r); + PASS(); +} + +/* Standalone CR ends a line comment, so a `?>` inside the next CR-only line's + * string is not a close tag; a `?>` that ends a `//` or `#` comment blanks the + * comment so the PHP reopened on that line is not swallowed; `';\rfunction a() {}\r"; + ASSERT(cbm_php_mask_inline_html(&arena, cr, (int)strlen(cr)) == cr); + + const char *cmt = "\n\n"; + masked = cbm_php_mask_inline_html(&arena, shorttag, (int)strlen(shorttag)); + ASSERT_STR_EQ(masked, " Int { 1 }\n" + "struct Box {\n" + " let value: Int\n" + " init() {\n" + " value = helper()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Box.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_STR_EQ(init_def->label, "Method"); + ASSERT_STR_EQ(init_def->name, "init"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a class's designated init takes parameters -- coverage that a + * parameterized init is extracted the same way as a bare one. */ +TEST(swift_init_class_designated_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = helper() + seed\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_STR_EQ(init_def->label, "Method"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a convenience init delegates via `self.init(...)`. Its call to + * `other()` (an argument of the delegating call) must source from the + * initializer, not from the file. */ +TEST(swift_init_convenience_delegates_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = seed\n" + " }\n" + " convenience init() {\n" + " self.init(seed: other())\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + + /* self.init(...) delegates to another overload of Holder.init, which + * shares this node (#2061): it forms no call, because the only edge it + * could produce is a self-loop. Its argument call stays. */ + ASSERT_FALSE(has_call(r, "self.init")); + ASSERT_FALSE(has_call(r, "Holder.init")); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: two initializers in the same type calling the SAME callee must both + * attribute to the type's init, not to the file's Module scope. Both share + * the QN `Holder.init` -- codebase-memory-mcp already collapses overloaded + * Swift methods onto one node (DeusData/codebase-memory-mcp#2061), and an + * init is no exception; distinguishing overloads by signature is that + * issue's concern, not this one's. */ +TEST(swift_init_overloads_share_one_node_issue2379) { + CBMFileResult *r = extract("func shared() -> Int { 3 }\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = shared()\n" + " }\n" + " convenience init() {\n" + " self.init(seed: shared())\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Holder.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Holder.init"); + ASSERT_NOT_NULL(init_def); + + int from_init = 0; + for (int i = 0; i < r->calls.count; i++) { + const CBMCall *c = &r->calls.items[i]; + if (!c->callee_name || strcmp(c->callee_name, "shared") != 0 || !c->enclosing_func_qn) { + continue; + } + ASSERT_STR_EQ(c->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(c->enclosing_func_qn, r->module_qn); + from_init++; + } + ASSERT_EQ(from_init, 2); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a failable `init?` in an enum is still a callable initializer. */ +TEST(swift_init_failable_enum_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "enum Mode {\n" + " case on\n" + " init?(flag: Bool) {\n" + " if helper() > 0 { self = .on } else { return nil }\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Mode.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Mode.init"); + ASSERT_NOT_NULL(init_def); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: `extension X { }` is modeled by the vendored grammar as a + * class_declaration named X, so an init declared in an extension goes through + * the same class-method path as one declared in the primary type. */ +TEST(swift_init_extension_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "extension Box {\n" + " init(raw: Int) {\n" + " value = other()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box+Raw.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Box.init"); + ASSERT_NOT_NULL(init_def); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: regression guard -- an ordinary method in the same type keeps its own + * attribution unchanged (this already worked pre-fix; the initializer fix + * must not disturb it). */ +TEST(swift_method_still_attributed_issue2379) { + CBMFileResult *r = extract("func other() -> Int { 2 }\n" + "struct Box {\n" + " init() {}\n" + " func read() -> Int {\n" + " other()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Box.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *read_def = find_def_in_module(r, "Box.read"); + ASSERT_NOT_NULL(read_def); + + const CBMCall *call = find_call_by_callee(r, "other"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, read_def->qualified_name); + + cbm_free_result(r); + PASS(); +} + +/* #2379: a `deinit` has the same gap as `init` -- no source-level name, so a + * call inside its body attributed to the file's Module scope instead of the + * type. */ +TEST(swift_deinit_issue2379) { + CBMFileResult *r = extract("func helper() -> Int { 1 }\n" + "final class Deinitable {\n" + " deinit {\n" + " _ = helper()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Deinitable.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *deinit_def = find_def_in_module(r, "Deinitable.deinit"); + ASSERT_NOT_NULL(deinit_def); + ASSERT_STR_EQ(deinit_def->name, "deinit"); + + const CBMCall *call = find_call_by_callee(r, "helper"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, deinit_def->qualified_name); + ASSERT_STR_NEQ(call->enclosing_func_qn, r->module_qn); + + cbm_free_result(r); + PASS(); +} + +/* #2379: `super.init(...)` names no type. Left as written, the registry binds it + * by short name to whichever project `init` is nearest (on Alamofire: a nested + * `Options.init`, a sibling decoder's). It is emitted as the superclass's + * initializer instead -- the first inheritance entry, generic arguments + * dropped -- and stays attributed to the calling initializer. */ +TEST(swift_super_init_names_superclass_issue2379) { + CBMFileResult *r = extract("final class Sub: Base, Sendable {\n" + " override init() {\n" + " super.init()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Sub.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + + const CBMDefinition *init_def = find_def_in_module(r, "Sub.init"); + ASSERT_NOT_NULL(init_def); + ASSERT_FALSE(has_call(r, "super.init")); + const CBMCall *call = find_call_by_callee(r, "Base.init"); + ASSERT_NOT_NULL(call); + ASSERT_NOT_NULL(call->enclosing_func_qn); + ASSERT_STR_EQ(call->enclosing_func_qn, init_def->qualified_name); + + cbm_free_result(r); + PASS(); +} + +/* #2379: without a superclass to name -- no inheritance clause, or a first entry + * that does not read as a type -- `super.init` forms no call at all. A + * lower-case root would pass the registry's receiver-chain check and bind by + * short name again. */ +TEST(swift_super_init_without_superclass_no_call_issue2379) { + CBMFileResult *r = extract("class Root {\n" + " init() {\n" + " super.init()\n" + " }\n" + "}\n" + "class Odd: base {\n" + " init() {\n" + " super.init()\n" + " }\n" + "}\n", + CBM_LANG_SWIFT, "t", "Root.swift"); + ASSERT_NOT_NULL(r); + ASSERT_NOT_NULL(find_def_in_module(r, "Root.init")); + ASSERT_NOT_NULL(find_def_in_module(r, "Odd.init")); + for (int i = 0; i < r->calls.count; i++) { + const char *callee = r->calls.items[i].callee_name; + ASSERT(!callee || !strstr(callee, "init")); + } + + cbm_free_result(r); + PASS(); +} + +/* #2379: an initializer call `T.init(...)` is flagged so the resolvers' + * weak-member guard keeps a short-name guess from placing it (nearly every + * type has an `init`); every other Swift call stays unflagged and resolves + * exactly as before. */ +TEST(swift_initializer_calls_flagged_issue2379) { + CBMFileResult *r = extract("func run(obj: Worker) {\n" + " _ = UIAlertAction.init(title: nil)\n" + " helper()\n" + " obj.start()\n" + "}\n", + CBM_LANG_SWIFT, "t", "Run.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + const CBMCall *init_call = find_call_by_callee(r, "UIAlertAction.init"); + const CBMCall *bare = find_call_by_callee(r, "helper"); + const CBMCall *member = find_call_by_callee(r, "obj.start"); + ASSERT_NOT_NULL(init_call); + ASSERT_NOT_NULL(bare); + ASSERT_NOT_NULL(member); + ASSERT_TRUE(init_call->is_method); + ASSERT_FALSE(bare->is_method); + ASSERT_FALSE(member->is_method); + + cbm_free_result(r); + PASS(); +} + +/* #2379: `init` is a Swift keyword. A reference to one (`.init(...)`, + * `Box.init`) names the initializer of a type the bare name does not carry, + * so it is not recorded as a usage -- by name it could only be guessed among + * every type's initializer (on Alamofire, 26 references landed on one nested + * `Output.init`). The type in `Box.init` is still a usage. */ +TEST(swift_init_reference_is_not_a_usage_issue2379) { + CBMFileResult *r = extract("struct Box {\n" + " init(raw: Int) {}\n" + "}\n" + "func make() -> Box {\n" + " .init(raw: 1)\n" + "}\n" + "func maker() -> (Int) -> Box {\n" + " Box.init\n" + "}\n", + CBM_LANG_SWIFT, "t", "Make.swift"); + ASSERT_NOT_NULL(r); + ASSERT_FALSE(r->has_error); + int init_usages = 0; + int box_usages = 0; + for (int i = 0; i < r->usages.count; i++) { + const char *ref = r->usages.items[i].ref_name; + init_usages += ref && strcmp(ref, "init") == 0; + box_usages += ref && strcmp(ref, "Box") == 0; + } + ASSERT_EQ(init_usages, 0); + ASSERT_GT(box_usages, 0); + + cbm_free_result(r); + PASS(); +} + /* --- Objective-C --- */ TEST(objc_interface) { CBMFileResult *r = @@ -7085,16 +7681,6 @@ TEST(extract_ts_decorators_survive_interleaved_comment) { PASS(); } -/* Find an in-body call by its raw callee text; returns the call or NULL. */ -static const CBMCall *find_call_by_callee(CBMFileResult *r, const char *callee) { - for (int i = 0; i < r->calls.count; i++) { - if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0) { - return &r->calls.items[i]; - } - } - return NULL; -} - /* #1892: the Swift grammar declares no "arguments" field, so the generic field * lookup read nothing and every Swift call lost its arguments. Without the URL * the service-pattern table cannot raise an HTTP_CALLS edge or a Route node, @@ -14887,6 +15473,13 @@ SUITE(extraction) { RUN_TEST(python_class_base_extracted_bare); RUN_TEST(php_class); RUN_TEST(php_function); + RUN_TEST(php_inline_html_tail_reaches_graph_issue2000); + RUN_TEST(php_inline_html_template_shapes_issue2000); + RUN_TEST(php_inline_html_close_tag_in_literals_issue2000); + RUN_TEST(php_inline_html_markup_never_reaches_graph_issue2000); + RUN_TEST(php_inline_html_mask_copies_only_on_write_issue2000); + RUN_TEST(php_inline_html_trailing_close_tag_crlf_issue2000); + RUN_TEST(php_inline_html_comment_close_tag_and_short_tag_masks); RUN_TEST(ruby_class); RUN_TEST(ruby_module); RUN_TEST(ruby_constructor_callee_is_constant_path); @@ -15013,6 +15606,18 @@ SUITE(extraction) { RUN_TEST(swift_constructor_call); RUN_TEST(swift_chained_call); RUN_TEST(swift_force_unwrap_scanner_shift); + RUN_TEST(swift_init_struct_issue2379); + RUN_TEST(swift_init_class_designated_issue2379); + RUN_TEST(swift_init_convenience_delegates_issue2379); + RUN_TEST(swift_init_overloads_share_one_node_issue2379); + RUN_TEST(swift_init_failable_enum_issue2379); + RUN_TEST(swift_init_extension_issue2379); + RUN_TEST(swift_method_still_attributed_issue2379); + RUN_TEST(swift_deinit_issue2379); + RUN_TEST(swift_super_init_names_superclass_issue2379); + RUN_TEST(swift_super_init_without_superclass_no_call_issue2379); + RUN_TEST(swift_initializer_calls_flagged_issue2379); + RUN_TEST(swift_init_reference_is_not_a_usage_issue2379); RUN_TEST(swift_call_string_arg_issue1892); RUN_TEST(swift_nested_url_constructor_issue1892); RUN_TEST(swift_nested_url_no_bang_issue1892); diff --git a/tests/test_index_policy.c b/tests/test_index_policy.c index c323d4c6a..4a47be422 100644 --- a/tests/test_index_policy.c +++ b/tests/test_index_policy.c @@ -21,9 +21,15 @@ TEST(index_policy_defaults_are_disabled) { ASSERT_FALSE(policy.max_files.enabled); ASSERT_FALSE(policy.max_source_bytes.enabled); + ASSERT_FALSE(policy.max_rss_bytes.enabled); + ASSERT_FALSE(policy.max_duration_ms.enabled); ASSERT_FALSE(cbm_index_policy_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_worker_enabled(&policy)); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_FILES), "off"); ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_SOURCE_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_RSS_MB), "off"); + ASSERT_STR_EQ(cbm_index_policy_default_value(CBM_INDEX_CONFIG_MAX_DURATION_SECONDS), "off"); PASS(); } @@ -60,6 +66,38 @@ TEST(index_policy_source_limit_converts_mib_without_overflow) { PASS(); } +TEST(index_policy_worker_limits_validate_and_convert_units) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + char error[256]; + + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE(cbm_index_policy_enabled(&policy)); + ASSERT_TRUE(cbm_index_policy_worker_enabled(&policy)); + ASSERT_FALSE(cbm_index_policy_discovery_enabled(&policy)); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "1048576", error, + sizeof(error))); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(1048576) * CBM_INDEX_MIB_BYTES); + cbm_index_resource_policy_t before = policy; + ASSERT_FALSE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "63", error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "1", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, 1000); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86400", error, + sizeof(error))); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(86400000)); + before = policy; + ASSERT_FALSE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "86401", + error, sizeof(error))); + ASSERT_EQ(memcmp(&policy, &before, sizeof(policy)), 0); + PASS(); +} + TEST(index_policy_invalid_value_is_rejected_atomically) { static const char *const invalid[] = {"", "0", "-1", "1MB", "1 ", "+1", "10000001", "18446744073709551616"}; @@ -95,6 +133,16 @@ TEST(index_policy_format_round_trips_public_values) { ASSERT_TRUE( cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_SOURCE_MB, value, sizeof(value))); ASSERT_STR_EQ(value, "7"); + ASSERT_TRUE( + cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, "64", error, sizeof(error))); + ASSERT_TRUE( + cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_RSS_MB, value, sizeof(value))); + ASSERT_STR_EQ(value, "64"); + ASSERT_TRUE(cbm_index_policy_set(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "9", error, + sizeof(error))); + ASSERT_TRUE(cbm_index_policy_format(&policy, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, value, + sizeof(value))); + ASSERT_STR_EQ(value, "9"); PASS(); } @@ -103,10 +151,18 @@ TEST(index_policy_violation_metadata_is_stable) { ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_SOURCE_BYTES), "source_bytes"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_FILES), "files"); ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_SOURCE_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_RSS_BYTES), "rss_bytes"); + ASSERT_STR_EQ(cbm_index_resource_name(CBM_INDEX_RESOURCE_DURATION_MS), "duration_ms"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_RSS_BYTES), "bytes"); + ASSERT_STR_EQ(cbm_index_resource_unit(CBM_INDEX_RESOURCE_DURATION_MS), "milliseconds"); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_FILES), CBM_INDEX_CONFIG_MAX_FILES); ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_SOURCE_BYTES), CBM_INDEX_CONFIG_MAX_SOURCE_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_RSS_BYTES), + CBM_INDEX_CONFIG_MAX_RSS_MB); + ASSERT_STR_EQ(cbm_index_resource_config_key(CBM_INDEX_RESOURCE_DURATION_MS), + CBM_INDEX_CONFIG_MAX_DURATION_SECONDS); PASS(); } @@ -122,9 +178,13 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { ASSERT_FALSE(cbm_index_policy_enabled(&policy)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "9"), 0); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_SOURCE_MB, "3"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_RSS_MB, "64"), 0); + ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS, "7"), 0); ASSERT_TRUE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); ASSERT_EQ(policy.max_files.value, 9); ASSERT_EQ(policy.max_source_bytes.value, UINT64_C(3) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_rss_bytes.value, UINT64_C(64) * CBM_INDEX_MIB_BYTES); + ASSERT_EQ(policy.max_duration_ms.value, UINT64_C(7000)); ASSERT_EQ(cbm_config_set(config, CBM_INDEX_CONFIG_MAX_FILES, "corrupt"), 0); ASSERT_FALSE(cbm_config_load_index_policy(config, &policy, error, sizeof(error))); @@ -135,16 +195,23 @@ TEST(index_policy_config_loads_defaults_values_and_rejects_corruption) { PASS(); } -TEST(index_policy_cli_lists_both_operator_keys) { +TEST(index_policy_cli_lists_all_operator_keys) { bool files_found = false; bool bytes_found = false; + bool rss_found = false; + bool duration_found = false; for (size_t index = 0; index < cbm_cli_config_key_count_for_testing(); index++) { const char *key = cbm_cli_config_key_at_for_testing(index); files_found = files_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_FILES) == 0); bytes_found = bytes_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_SOURCE_MB) == 0); + rss_found = rss_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_RSS_MB) == 0); + duration_found = + duration_found || (key && strcmp(key, CBM_INDEX_CONFIG_MAX_DURATION_SECONDS) == 0); } ASSERT_TRUE(files_found); ASSERT_TRUE(bytes_found); + ASSERT_TRUE(rss_found); + ASSERT_TRUE(duration_found); PASS(); } @@ -395,11 +462,12 @@ SUITE(index_policy) { RUN_TEST(index_policy_defaults_are_disabled); RUN_TEST(index_policy_file_limit_accepts_off_and_exact_range); RUN_TEST(index_policy_source_limit_converts_mib_without_overflow); + RUN_TEST(index_policy_worker_limits_validate_and_convert_units); RUN_TEST(index_policy_invalid_value_is_rejected_atomically); RUN_TEST(index_policy_format_round_trips_public_values); RUN_TEST(index_policy_violation_metadata_is_stable); RUN_TEST(index_policy_config_loads_defaults_values_and_rejects_corruption); - RUN_TEST(index_policy_cli_lists_both_operator_keys); + RUN_TEST(index_policy_cli_lists_all_operator_keys); RUN_TEST(index_policy_cli_set_rejects_invalid_value_without_overwrite); RUN_TEST(index_policy_cli_set_reports_a_failed_write); RUN_TEST(index_policy_worker_rejects_missing_parent_policy); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 52734c498..2239924fd 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -12,6 +12,7 @@ #include "operations/index_supervisor.h" #include "foundation/constants.h" #include "operations/index.h" +#include "operations/result_wire.h" #include #include @@ -239,18 +240,59 @@ TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar) { "/tmp/m", CBM_INDEX_WORKER_QUARANTINE_ARG, "/tmp/q", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "/tmp/f", NULL, }; cbm_index_worker_invocation_t invocation; - ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + ASSERT_EQ(cbm_index_worker_parse_process_argv(18, valid, &invocation), CBM_INDEX_WORKER_ARGV_VALID); - ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_EQ(cbm_daemon_process_role(18, valid), CBM_DAEMON_PROCESS_WORKER); ASSERT_STR_EQ(invocation.args_json, valid[6]); ASSERT_STR_EQ(invocation.response_out, valid[8]); ASSERT_EQ(invocation.memory_budget_bytes, 1024); ASSERT_TRUE(invocation.single_thread); ASSERT_STR_EQ(invocation.marker_file, valid[13]); ASSERT_STR_EQ(invocation.quarantine_file, valid[15]); + ASSERT_STR_EQ(invocation.publish_fence_file, valid[17]); + /* Without a worker limit there is no fence: the shorter grammar stays valid. */ + ASSERT_EQ(cbm_index_worker_parse_process_argv(16, valid, &invocation), + CBM_INDEX_WORKER_ARGV_VALID); + ASSERT_EQ(cbm_daemon_process_role(16, valid), CBM_DAEMON_PROCESS_WORKER); + ASSERT_NULL(invocation.publish_fence_file); + + char *fence_before_quarantine[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "/tmp/f", + CBM_INDEX_WORKER_QUARANTINE_ARG, + "/tmp/q", + NULL}; + char *empty_fence[] = {"test-runner", + "cli", + "--index-worker", + CBM_INDEX_WORKER_BUILD_ARG, + (char *)captured, + "index_repository", + "{}", + "--response-out", + "/tmp/r", + CBM_INDEX_WORKER_PUBLISH_FENCE_ARG, + "", + NULL}; + ASSERT_EQ(cbm_index_worker_parse_process_argv(13, fence_before_quarantine, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(13, fence_before_quarantine), CBM_DAEMON_PROCESS_INVALID); + ASSERT_EQ(cbm_index_worker_parse_process_argv(11, empty_fence, &invocation), + CBM_INDEX_WORKER_ARGV_INVALID); + ASSERT_EQ(cbm_daemon_process_role(11, empty_fence), CBM_DAEMON_PROCESS_INVALID); char *missing_build[] = {"test-runner", "cli", "--index-worker", "index_repository", "{}", "--response-out", @@ -1032,6 +1074,632 @@ TEST(index_supervisor_unreadable_response_reports_supervisor_error) { return index_supervisor_no_response_case(true); } +typedef struct { + uint64_t now_ms; + uint64_t rss_values[4]; + int rss_value_count; + int rss_calls; + int clock_calls; + cbm_proc_tree_rss_status_t rss_status; +} index_supervisor_resource_fake_t; + +static uint64_t index_supervisor_fake_clock(void *context) { + index_supervisor_resource_fake_t *fake = context; + fake->clock_calls++; + return fake->now_ms; +} + +static cbm_proc_tree_rss_status_t index_supervisor_fake_charged_rss(cbm_subprocess_t *process, + uint64_t *rss_bytes, + void *context) { + (void)process; + index_supervisor_resource_fake_t *fake = context; + int value_index = + fake->rss_calls < fake->rss_value_count ? fake->rss_calls : fake->rss_value_count - 1; + fake->rss_calls++; + if (fake->rss_status == CBM_PROC_TREE_RSS_OK && rss_bytes && value_index >= 0) { + *rss_bytes = fake->rss_values[value_index]; + } + return fake->rss_status; +} + +static cbm_index_resource_policy_t index_supervisor_test_worker_policy(uint64_t rss_bytes, + uint64_t duration_ms) { + cbm_index_resource_policy_t policy; + cbm_index_policy_init(&policy); + policy.max_rss_bytes = (cbm_index_limit_u64_t){.enabled = rss_bytes > 0, .value = rss_bytes}; + policy.max_duration_ms = + (cbm_index_limit_u64_t){.enabled = duration_ms > 0, .value = duration_ms}; + return policy; +} + +TEST(index_supervisor_disabled_limits_do_not_probe) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {UINT64_MAX}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"clean\"}", 0, + &policy, false, NULL, NULL, 0, &handle); + /* Opt-in: no worker limit, no publication fence, no claim. */ + bool fenced = !handle || cbm_index_worker_publish_fence_path_for_testing(handle) != NULL; + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + cbm_index_resource_t resource = + result ? result->resource_violation.resource : CBM_INDEX_RESOURCE_RSS_BYTES; + bool probe_failed = result && result->resource_probe_failed; + bool committed = !result || result->publication_committed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_EQ(fake.rss_calls, 0); + ASSERT_EQ(fake.clock_calls, 0); + ASSERT_EQ(resource, CBM_INDEX_RESOURCE_NONE); + ASSERT_FALSE(probe_failed); + ASSERT_FALSE(fenced); + ASSERT_FALSE(committed); + PASS(); +} + +TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit, limit + 1}, + .rss_value_count = 2, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 0, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + const cbm_index_worker_result_t *cached = NULL; + bool cached_terminal = + terminal && cbm_index_worker_poll(handle, &cached) == CBM_INDEX_WORKER_POLL_TERMINAL && + cached == result; + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + result->resource_violation.observed == limit + 1 && + result->resource_violation.limit == limit && !result->cancellation_requested && + result->tree_quiesced && !result->supervision_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + ASSERT_TRUE(cached_terminal); + PASS(); +} + +TEST(index_supervisor_duration_is_total_time_not_quiet_timeout) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 0, &handle); + fake.now_ms = 1100; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t equal_state = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms = 1101; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && + result->resource_violation.limit == 1000 && result->outcome != CBM_PROC_HANG && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(equal_state, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_duration_spans_recovery_attempts) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *first = NULL; + int first_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &first); + fake.now_ms = 500; + const cbm_index_worker_result_t *first_result = NULL; + cbm_index_worker_poll_t mid = + first ? cbm_index_worker_poll(first, &first_result) : CBM_INDEX_WORKER_POLL_ERROR; + bool cancel_accepted = first && cbm_index_worker_request_cancel(first); + bool first_terminal = first && index_supervisor_test_poll_terminal( + first, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &first_result); + if (first_terminal) { + cbm_index_worker_destroy(first); + first = NULL; + } else { + index_supervisor_test_cleanup_handle(first); + first = NULL; + } + + fake.now_ms = 1101; + cbm_index_worker_handle_t *second = NULL; + int second_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &second); + const cbm_index_worker_result_t *second_result = NULL; + bool second_terminal = second && index_supervisor_test_poll_terminal( + second, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &second_result); + bool limited = second_terminal && second_result && + second_result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + second_result->resource_violation.observed == 1001 && + second_result->resource_violation.limit == 1000 && + second_result->outcome != CBM_PROC_HANG && second_result->tree_quiesced; + if (second_terminal) { + cbm_index_worker_destroy(second); + } else { + index_supervisor_test_cleanup_handle(second); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(first_rc, 0); + ASSERT_EQ(mid, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(first_terminal); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(limited); + PASS(); +} + +TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled) { + const char *saved_timeout = getenv("CBM_INDEX_WORKER_TIMEOUT_S"); + char *saved_timeout_copy = saved_timeout ? cbm_strdup(saved_timeout) : NULL; + (void)cbm_setenv("CBM_INDEX_WORKER_TIMEOUT_S", "1", 1); + /* The duration watchdog stays armed, but its clock is pinned at the request + * origin, so the quiet timeout (the subprocess's own clock) is the only way + * this hung worker can end, however slowly the host runs. With the real + * clock a starved run could cross the duration limit first and turn the + * HANG into a resource kill. The limit equals the quiet timeout on purpose: + * more real time than the limit always passes, and it still cannot fire. */ + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 100, &handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool remained_hang = terminal && result && result->outcome == CBM_PROC_HANG && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + index_supervisor_test_restore_env("CBM_INDEX_WORKER_TIMEOUT_S", saved_timeout_copy); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(remained_hang); + /* The duration check really ran while polling (it read the pinned clock), + * so HANG won against a live watchdog, not against a disabled one. */ + ASSERT_TRUE(fake.clock_calls > 0); + PASS(); +} + +TEST(index_supervisor_cancel_precedes_resource_probe) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_values = {limit + 1}, + .rss_value_count = 1, + .rss_status = CBM_PROC_TREE_RSS_OK, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 0, &handle); + fake.now_ms = 2000; + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool cancelled = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(cancelled); + ASSERT_EQ(fake.rss_calls, 0); + PASS(); +} + +TEST(index_supervisor_three_failed_rss_probes_fail_closed) { + const uint64_t limit = UINT64_C(64) * CBM_INDEX_MIB_BYTES; + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_ERROR, + }; + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(limit, 0); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, NULL, NULL, 0, &handle); + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t first = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + cbm_index_worker_poll_t throttled = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int calls_before_interval = fake.rss_calls; + fake.now_ms += 250; + cbm_index_worker_poll_t second = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + fake.now_ms += 250; + cbm_index_worker_poll_t third = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool failed_closed = terminal && result && result->resource_probe_failed && + result->resource_violation.resource == CBM_INDEX_RESOURCE_RSS_BYTES && + !result->cancellation_requested && result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + + ASSERT_EQ(start_rc, 0); + ASSERT_EQ(first, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(throttled, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(calls_before_interval, 1); + ASSERT_EQ(second, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(third, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(failed_closed); + ASSERT_EQ(fake.rss_calls, 3); + PASS(); +} + +/* Publication fence (index_supervisor.h). The hang-tree probe reports the fence + * path it received on its argv in its marker, and the test plays the worker's + * publication claim through the production entry point with that path. */ +typedef struct { + char directory[INDEX_SUPERVISOR_TEST_PATH_CAP]; + char marker[INDEX_SUPERVISOR_TEST_PATH_CAP]; + char fence[INDEX_SUPERVISOR_TEST_PATH_CAP]; +} index_supervisor_fence_fixture_t; + +static bool index_supervisor_fence_fixture_init(index_supervisor_fence_fixture_t *fixture) { + memset(fixture, 0, sizeof(*fixture)); + (void)snprintf(fixture->directory, sizeof(fixture->directory), "%s/cbm-index-fence-XXXXXX", + cbm_tmpdir()); + if (!cbm_mkdtemp(fixture->directory)) { + return false; + } + (void)snprintf(fixture->marker, sizeof(fixture->marker), "%s/marker", fixture->directory); + return true; +} + +/* True once the worker started with exactly this attempt's fence. */ +static bool index_supervisor_fence_fixture_started(index_supervisor_fence_fixture_t *fixture, + cbm_index_worker_handle_t *handle) { + const char *fence = handle ? cbm_index_worker_publish_fence_path_for_testing(handle) : NULL; + if (!fence) { + return false; + } + (void)snprintf(fixture->fence, sizeof(fixture->fence), "%s", fence); + char expected[INDEX_SUPERVISOR_TEST_PATH_CAP + 16]; + (void)snprintf(expected, sizeof(expected), "fence=%s\n", fixture->fence); + return index_supervisor_test_wait_file_text(fixture->marker, expected, + INDEX_SUPERVISOR_TEST_READY_MS); +} + +static bool index_supervisor_fence_fixture_worker_claims( + index_supervisor_fence_fixture_t *fixture) { + cbm_index_set_worker_publish_fence(fixture->fence); + bool claimed = cbm_index_worker_claim_publication(); + cbm_index_set_worker_publish_fence(NULL); + return claimed; +} + +static void index_supervisor_fence_fixture_cleanup(index_supervisor_fence_fixture_t *fixture) { + (void)cbm_unlink(fixture->marker); + (void)cbm_rmdir(fixture->directory); +} + +static int g_index_supervisor_after_publication_logs; + +static void index_supervisor_test_after_publication_sink(const char *line) { + if (line && strstr(line, "index.supervisor.resource_limit_after_publication")) { + g_index_supervisor_after_publication_logs++; + } +} + +TEST(index_supervisor_limit_claims_publication_before_terminating) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + index_supervisor_fence_fixture_t fixture; + bool fixture_ready = index_supervisor_fence_fixture_init(&fixture); + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + fixture_ready + ? cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, fixture.marker, NULL, 100, &handle) + : -1; + bool started = start_rc == 0 && index_supervisor_fence_fixture_started(&fixture, handle); + /* The limit is crossed while the worker has not claimed publication: the + * supervisor claims the fence in this poll, before it stops the worker. */ + fake.now_ms = 1101; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t decided = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + /* Whatever the worker does next, it can no longer publish. */ + bool worker_barred = started && !index_supervisor_fence_fixture_worker_claims(&fixture); + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool limited = terminal && result && + result->resource_violation.resource == CBM_INDEX_RESOURCE_DURATION_MS && + result->resource_violation.observed == 1001 && !result->publication_committed && + !result->cancellation_requested && result->tree_quiesced; + bool fence_removed = terminal && fixture.fence[0] && cbm_file_size(fixture.fence) < 0; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + if (!started) { + index_supervisor_test_dump("marker", fixture.marker); + } + index_supervisor_fence_fixture_cleanup(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(started); + ASSERT_EQ(decided, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(worker_barred); + ASSERT_TRUE(limited); + ASSERT_TRUE(fence_removed); + PASS(); +} + +TEST(index_supervisor_limit_after_worker_publication_claim_is_not_enforced) { + index_supervisor_resource_fake_t fake = { + .now_ms = 100, + .rss_status = CBM_PROC_TREE_RSS_EMPTY, + }; + index_supervisor_fence_fixture_t fixture; + bool fixture_ready = index_supervisor_fence_fixture_init(&fixture); + CBMLogLevel saved_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_INFO); + g_index_supervisor_after_publication_logs = 0; + cbm_log_set_sink_ex(index_supervisor_test_after_publication_sink, CBM_LOG_SINK_TEE); + cbm_index_supervisor_set_resource_hooks_for_testing(index_supervisor_fake_clock, + index_supervisor_fake_charged_rss, &fake); + cbm_index_resource_policy_t policy = index_supervisor_test_worker_policy(0, 1000); + cbm_index_worker_handle_t *handle = NULL; + int start_rc = + fixture_ready + ? cbm_index_worker_start_with_policy("{\"__cbm_test_worker\":\"hang-tree\"}", 0, + &policy, false, fixture.marker, NULL, 100, &handle) + : -1; + bool started = start_rc == 0 && index_supervisor_fence_fixture_started(&fixture, handle); + /* The worker commits to publishing first; only then is the limit crossed. */ + bool worker_claimed = started && index_supervisor_fence_fixture_worker_claims(&fixture); + fake.now_ms = 1101; + const cbm_index_worker_result_t *result = NULL; + cbm_index_worker_poll_t decided = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + int clock_calls_at_decision = fake.clock_calls; + cbm_index_worker_poll_t after = + handle ? cbm_index_worker_poll(handle, &result) : CBM_INDEX_WORKER_POLL_ERROR; + bool disarmed = fake.clock_calls == clock_calls_at_decision; + /* The attempt runs on; the test ends it with an ordinary cancellation. */ + bool cancel_accepted = handle && cbm_index_worker_request_cancel(handle); + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool committed = terminal && result && result->cancellation_requested && + result->resource_violation.resource == CBM_INDEX_RESOURCE_NONE && + !result->resource_probe_failed && result->publication_committed && + result->tree_quiesced; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_cleanup_handle(handle); + } + cbm_index_supervisor_reset_resource_hooks_for_testing(); + cbm_log_set_sink(NULL); + cbm_log_set_level(saved_level); + if (!started) { + index_supervisor_test_dump("marker", fixture.marker); + } + index_supervisor_fence_fixture_cleanup(&fixture); + + ASSERT_TRUE(fixture_ready); + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(started); + ASSERT_TRUE(worker_claimed); + ASSERT_EQ(decided, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_EQ(after, CBM_INDEX_WORKER_POLL_RUNNING); + ASSERT_TRUE(disarmed); + ASSERT_EQ(g_index_supervisor_after_publication_logs, 1); + ASSERT_TRUE(cancel_accepted); + ASSERT_TRUE(committed); + PASS(); +} + +/* The resource response travels as the native result wire; return its decoded + * payload JSON (error bit required), caller frees. */ +static char *index_supervisor_resource_payload(const char *args, + const cbm_index_worker_result_t *result, + const cbm_index_request_facts_t *facts) { + char *wire = cbm_index_worker_resource_response(args, result, facts); + cbm_operation_result_t decoded = {0}; + char *payload = NULL; + if (wire && cbm_operation_result_wire_decode(wire, &decoded) && decoded.is_error && + decoded.payload) { + payload = strdup(decoded.payload); + } + cbm_operation_result_dispose(&decoded); + free(wire); + return payload; +} + +static bool index_supervisor_request_mutation_begin(void *context, const char *project) { + (void)context; + (void)project; + return true; +} + +static void index_supervisor_request_mutation_end(void *context, const char *project) { + (void)context; + (void)project; +} + +/* serving_index_preserved describes the request, not only its last attempt: + * an earlier attempt may have published before it crashed (recovery then runs + * another attempt), and an index that appeared during the request (artifact + * bootstrap) was not preserved. */ +TEST(index_supervisor_resource_response_reports_preservation_for_the_request) { + const char *made = th_mktempdir("cbm_request_facts_repo"); + char *repo = made ? strdup(made) : NULL; + made = th_mktempdir("cbm_request_facts_cache"); + char *cache = made ? strdup(made) : NULL; + ASSERT_NOT_NULL(repo); + ASSERT_NOT_NULL(cache); + ASSERT_EQ(th_write_file(TH_PATH(repo, "main.c"), "int main(void) { return 0; }\n"), 0); + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? strdup(old_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + char args[CBM_SZ_2K]; + (void)snprintf(args, sizeof(args), + "{\"repo_path\":\"%s\",\"mode\":\"fast\",\"name\":\"RequestFactsFixture\"," + "\"_cbm_index_policy\":{\"index_max_files\":\"off\"," + "\"index_max_source_mb\":\"off\",\"index_max_rss_mb\":\"off\"," + "\"index_max_duration_seconds\":\"off\"}}", + repo); + + cbm_index_request_facts_t before_index = {0}; + cbm_index_request_facts_begin(args, &before_index); + cbm_operation_runtime_t runtime = {0}; + runtime.mutation_begin = index_supervisor_request_mutation_begin; + runtime.mutation_try_begin = index_supervisor_request_mutation_begin; + runtime.mutation_end = index_supervisor_request_mutation_end; + cbm_operation_result_t indexed = cbm_index_operation_execute(args, &runtime); + bool indexed_ok = indexed.payload && !indexed.is_error; + cbm_operation_result_dispose(&indexed); + cbm_index_request_facts_t facts = {0}; + cbm_index_request_facts_begin(args, &facts); + + cbm_index_worker_result_t limited; + memset(&limited, 0, sizeof(limited)); + limited.outcome = CBM_PROC_KILLED; + limited.tree_quiesced = true; + limited.resource_violation = (cbm_index_resource_violation_t){ + .resource = CBM_INDEX_RESOURCE_DURATION_MS, .observed = 1001, .limit = 1000}; + cbm_index_worker_result_t published_then_crashed; + memset(&published_then_crashed, 0, sizeof(published_then_crashed)); + published_then_crashed.outcome = CBM_PROC_CRASH; + published_then_crashed.publication_committed = true; + cbm_index_request_facts_t after_recovery = facts; + cbm_index_request_facts_record(&after_recovery, &limited); + bool limited_attempt_commits_nothing = !after_recovery.publication_committed; + cbm_index_request_facts_record(&after_recovery, &published_then_crashed); + + char *preserved = index_supervisor_resource_payload(args, &limited, &facts); + char *earlier_published = index_supervisor_resource_payload(args, &limited, &after_recovery); + char *appeared = index_supervisor_resource_payload(args, &limited, &before_index); + char *no_facts = index_supervisor_resource_payload(args, &limited, NULL); + const char *kept = "\"serving_index_preserved\":true"; + const char *not_kept = "\"serving_index_preserved\":false"; + bool preserved_ok = + preserved && strstr(preserved, kept) && strstr(preserved, "resource_limit_exceeded") && + strstr(preserved, "\"stage\":\"worker\"") && strstr(preserved, "\"observed\":1001") && + strstr(preserved, "\"limit\":1000"); + bool earlier_ok = earlier_published && strstr(earlier_published, not_kept); + bool appeared_ok = appeared && strstr(appeared, not_kept); + bool no_facts_ok = no_facts && strstr(no_facts, not_kept); + + free(preserved); + free(earlier_published); + free(appeared); + free(no_facts); + index_supervisor_test_restore_env("CBM_CACHE_DIR", saved_cache); + th_cleanup(repo); + th_cleanup(cache); + free(repo); + free(cache); + + ASSERT_FALSE(before_index.serving_index_existed); + ASSERT_TRUE(indexed_ok); + ASSERT_TRUE(facts.serving_index_existed); + ASSERT_FALSE(facts.publication_committed); + ASSERT_TRUE(limited_attempt_commits_nothing); + ASSERT_TRUE(after_recovery.publication_committed); + ASSERT_TRUE(preserved_ok); + ASSERT_TRUE(earlier_ok); + ASSERT_TRUE(appeared_ok); + ASSERT_TRUE(no_facts_ok); + PASS(); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -1044,4 +1712,14 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); RUN_TEST(index_supervisor_unreadable_response_reports_supervisor_error); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); + RUN_TEST(index_supervisor_disabled_limits_do_not_probe); + RUN_TEST(index_supervisor_charged_rss_equality_runs_then_excess_terminates_tree); + RUN_TEST(index_supervisor_duration_is_total_time_not_quiet_timeout); + RUN_TEST(index_supervisor_duration_spans_recovery_attempts); + RUN_TEST(index_supervisor_quiet_timeout_remains_hang_with_duration_enabled); + RUN_TEST(index_supervisor_cancel_precedes_resource_probe); + RUN_TEST(index_supervisor_three_failed_rss_probes_fail_closed); + RUN_TEST(index_supervisor_limit_claims_publication_before_terminating); + RUN_TEST(index_supervisor_limit_after_worker_publication_claim_is_not_enforced); + RUN_TEST(index_supervisor_resource_response_reports_preservation_for_the_request); } diff --git a/tests/test_lint_mem_gate.sh b/tests/test_lint_mem_gate.sh new file mode 100755 index 000000000..9178bca56 --- /dev/null +++ b/tests/test_lint_mem_gate.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Regression contract for clang-tidy's `,-warnings-as-errors` diagnostic suffix +# (#2442). With WarningsAsErrors enabled, clang-tidy prints an analyzer check as +# [clang-analyzer-,-warnings-as-errors]; a gate that only accepts +# [] drops every such line and reports a clean gate. Both directions are +# pinned, and every assertion is reported, so a regression names what broke: +# 1. a suffixed finding WITH an argued whitelist entry is counted under its +# plain check name and suppressed (exit 0, "1 argued false positive"); +# 2. a suffixed finding WITHOUT an entry fails the gate (exit 1) and is +# reported under its plain check name. This is the negative assertion: a +# parser that drops the suffixed line reports clean (exit 0) here. +# Reverting the suffix group in FINDING_RE (scripts/lint-mem-gate.py) turns +# both red. +# +# The gate runs from inside the fixture directory with RELATIVE paths. On the +# Windows leg python3 is a native MSYS2 build: MSYS converts path ARGUMENTS for +# it, but never the diagnostic text on stdin, so an absolute /c/... path in a +# finding would not resolve there, and a C:/... one would not parse (the file +# field ends at the first colon). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GATE="$ROOT/scripts/lint-mem-gate.py" +FIX="$(mktemp -d "${TMPDIR:-/tmp}/lint-mem-gate.XXXXXX")" +trap 'rm -rf "$FIX"' EXIT + +failures=0 +fail() { + echo "FAIL: $*" >&2 + failures=$((failures + 1)) +} + +show() { + sed 's/^/ | /' "$1" >&2 +} + +# A subshell, so the test's own shell never sits inside $FIX (Windows cannot +# remove a directory that is a process's working directory). +run_gate() { + (cd "$FIX" && LINT_MEM_WHITELIST=whitelist.txt python3 "$GATE" "$@") +} + +cat > "$FIX/sample.c" <<'EOF' +int sample(int *value) { + return *value; +} + +int unlisted(int *value) { + return *value; +} +EOF + +# A native Windows python3 writes CRLF; strip it before the hash is reused. +HASH="$(run_gate --hash sample.c sample | tr -d '\r' | awk '/^segment-sha256:/ {print $2}')" +if [[ ! "$HASH" =~ ^[0-9a-f]{64}$ ]]; then + echo "FAIL: lint-mem-gate.py --hash sample.c sample printed no segment hash (got '$HASH')" >&2 + exit 1 +fi + +cat > "$FIX/whitelist.txt" < /dev/null 2> "$FIX/whitelisted.err" || rc=$? +if [ "$rc" -ne 0 ]; then + fail "a whitelisted suffixed finding left the gate at exit $rc, expected 0" + show "$FIX/whitelisted.err" +fi +if ! grep -q 'memory gate clean (1 argued false positive' "$FIX/whitelisted.err"; then + fail "a whitelisted suffixed finding was not counted as 1 argued false positive" + show "$FIX/whitelisted.err" +fi + +# 2. Not whitelisted: the gate must fail and name the finding. +rc=0 +printf '%s\n' "sample.c:6:12: warning: possible null dereference $SUFFIX" \ + | run_gate > /dev/null 2> "$FIX/unlisted.err" || rc=$? +if [ "$rc" -ne 1 ]; then + fail "an unwhitelisted suffixed finding left the gate at exit $rc, expected 1" \ + "(exit 0 = the suffixed line was parsed as no finding, #2442)" + show "$FIX/unlisted.err" +fi +if ! grep -qF 'sample.c:6: possible null dereference [clang-analyzer-core.NullDereference]' "$FIX/unlisted.err" || + ! grep -qF 'no whitelist entry for unlisted()' "$FIX/unlisted.err" || + ! grep -qF '1 finding(s) unaccounted for' "$FIX/unlisted.err"; then + fail "the gate did not report the unwhitelisted suffixed finding under its plain check name" + show "$FIX/unlisted.err" +fi + +if [ "$failures" -ne 0 ]; then + echo "lint-mem-gate warning suffix contract: $failures assertion(s) failed" >&2 + exit 1 +fi +echo 'lint-mem-gate warning suffix contract passed (whitelisted + unwhitelisted suffixed findings)' diff --git a/tests/test_main.c b/tests/test_main.c index 33425399e..0f36c9a7c 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -45,6 +45,7 @@ int tf_deselected_count = 0; #include #include "foundation/win_utf8.h" /* cbm_module_path_utf8 — the runner's own image */ #else +#include /* waitpid: the process-tree probe root */ #include #ifdef __APPLE__ #include @@ -460,10 +461,12 @@ static void tf_index_worker_probe(const char *args_json, const char *response_ou FILE *ready = marker ? cbm_fopen(marker, "wb") : NULL; if (ready) { (void)fprintf( - ready, "single=%s\nmarker=%s\nquarantine=%s\nbudget=%zu\ndescendant=%ld\n", + ready, + "single=%s\nmarker=%s\nquarantine=%s\nbudget=%zu\ndescendant=%ld\nfence=%s\n", getenv("CBM_INDEX_SINGLE_THREAD") ? getenv("CBM_INDEX_SINGLE_THREAD") : "", marker, getenv("CBM_INDEX_QUARANTINE_FILE") ? getenv("CBM_INDEX_QUARANTINE_FILE") : "", - cbm_mem_budget(), descendant); + cbm_mem_budget(), descendant, + cbm_index_worker_publish_fence() ? cbm_index_worker_publish_fence() : ""); (void)fclose(ready); } (void)fprintf(stderr, "async worker hang-tree probe\n"); @@ -514,6 +517,7 @@ static int tf_maybe_run_index_worker(int argc, char **argv) { cbm_index_set_worker_role_options(true, invocation.response_out, invocation.single_thread, invocation.marker_file, invocation.quarantine_file, invocation.memory_budget_bytes); + cbm_index_set_worker_publish_fence(invocation.publish_fence_file); cbm_mem_init_with_cap(0.5, invocation.memory_budget_bytes); cbm_log_init_for_process(false, true); tf_index_worker_probe(invocation.args_json, invocation.response_out); @@ -2034,6 +2038,116 @@ static int tf_role_env_isolation(int argc, char **argv) { return 0; } +/* Process-tree memory probe coverage (test_subprocess.c). The root starts ONE + * descendant that charges a controlled amount of memory, so a tree total can + * be checked against an allocation only a descendant holds. The descendant + * publishes its pid by an atomic rename once every page is touched, then + * blocks until it is killed. The root exits when its descendant dies, so a + * failed setup ends the tree instead of leaving the test waiting. */ +static int tf_rss_tree_hold(const char *ready_path, const char *mib_text, bool own_group) { + char *end = NULL; + long mib = strtol(mib_text, &end, 10); + if (!end || *end != '\0' || mib < 0 || mib > 1024) { + return 81; + } + size_t bytes = (size_t)mib * 1024U * 1024U; + unsigned char *block = bytes > 0 ? malloc(bytes) : NULL; + if (bytes > 0 && !block) { + return 82; + } + if (block) { + memset(block, 0xA5, bytes); + } + char staged[1024]; + int length = snprintf(staged, sizeof(staged), "%s.tmp", ready_path); + FILE *ready = length > 0 && (size_t)length < sizeof(staged) ? cbm_fopen(staged, "wb") : NULL; + if (!ready) { + return 83; + } +#ifdef _WIN32 + (void)fprintf(ready, "%lu\n", (unsigned long)GetCurrentProcessId()); +#else + (void)fprintf(ready, "%ld\n", (long)getpid()); +#endif + if (fclose(ready) != 0 || rename(staged, ready_path) != 0) { + return 84; + } + volatile unsigned char keep = block ? block[bytes - 1] : 0; + (void)keep; +#ifndef _WIN32 + if (own_group) { + /* Nothing kills a process outside the worker's group, so leave when + * the root (the parent) is gone, with an alarm as a backstop. */ + pid_t parent = getppid(); + (void)alarm(120); + while (getppid() == parent) { + cbm_usleep(10000); + } + return 0; + } +#endif + for (;;) { +#ifdef _WIN32 + Sleep(INFINITE); +#else + (void)pause(); +#endif + } +} + +static int tf_role_rss_tree_hold(int argc, char **argv) { + return argc == 4 ? tf_rss_tree_hold(argv[2], argv[3], false) : -1; +} + +static int tf_role_rss_tree_root(int argc, char **argv) { + /* A fifth argument "detach" puts the descendant in a process group of its + * own, as git discovery children are. */ + if (argc != 4 && argc != 5) { + return -1; + } +#ifdef _WIN32 + char *self = cbm_module_path_utf8(); + const char *child_argv[] = {self, "__cbm_rss_tree_hold", argv[2], argv[3], NULL}; + char command_line[8192]; + bool built = self && cbm_build_win_cmdline(command_line, sizeof(command_line), child_argv); + wchar_t *wide_self = built ? cbm_utf8_to_wide(self) : NULL; + wchar_t *wide_command = built ? cbm_utf8_to_wide(command_line) : NULL; + STARTUPINFOW startup; + ZeroMemory(&startup, sizeof(startup)); + startup.cb = sizeof(startup); + PROCESS_INFORMATION child; + ZeroMemory(&child, sizeof(child)); + /* The child joins this process's Job Object by default, which is the tree + * the probe under test enumerates. */ + BOOL started = + wide_self && wide_command && + CreateProcessW(wide_self, wide_command, NULL, NULL, FALSE, 0, NULL, NULL, &startup, &child); + free(wide_self); + free(wide_command); + free(self); + if (!started) { + return 85; + } + CloseHandle(child.hThread); + (void)WaitForSingleObject(child.hProcess, INFINITE); + CloseHandle(child.hProcess); +#else + pid_t child = fork(); + if (child < 0) { + return 85; + } + if (child == 0) { + if (argc == 5 && setpgid(0, 0) != 0) { + _exit(86); + } + _exit(tf_rss_tree_hold(argv[2], argv[3], argc == 5)); + } + int status = 0; + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} +#endif + return 86; /* the descendant ended: end the tree */ +} + /* Installation tests use this executable as a structurally real candidate. * Mirror the production binary's minimal verification contract. */ static int tf_role_version(int argc, char **argv) { @@ -2127,6 +2241,8 @@ typedef struct { static const tf_role_t tf_roles[] = { TF_ROLE(TF_PHASE_RAW, "__cbm_env_values", tf_role_env_values), TF_ROLE(TF_PHASE_RAW, "__cbm_env_isolation", tf_role_env_isolation), + TF_ROLE(TF_PHASE_RAW, "__cbm_rss_tree_root", tf_role_rss_tree_root), + TF_ROLE(TF_PHASE_RAW, "__cbm_rss_tree_hold", tf_role_rss_tree_hold), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdin_probe", tf_role_subprocess_stdin_probe), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_subprocess_stdout_probe", tf_role_subprocess_stdout_probe), TF_ROLE(TF_PHASE_GIT_CLEAN, "__cbm_git_facts_diff_probe", tf_role_git_facts_diff_probe), diff --git a/tests/test_php_lsp.c b/tests/test_php_lsp.c index 50586980c..edeff578c 100644 --- a/tests/test_php_lsp.c +++ b/tests/test_php_lsp.c @@ -112,6 +112,65 @@ TEST(phplsp_local_method_via_typed_param) { PASS(); } +/* #2000: with no cached tree (a result spilled under the memory budget comes + * back without one) the cross-file pass re-parses the on-disk source itself. + * It must blank inline HTML exactly as the per-file extract does, or a file's + * tail would lose cross-file resolution only when that file spilled. */ +static int find_resolved_arr(const CBMResolvedCallArray *arr, const char *callerSub, + const char *calleeSub) { + for (int i = 0; i < arr->count; i++) { + const CBMResolvedCall *rc = &arr->items[i]; + if (rc->caller_qn && strstr(rc->caller_qn, callerSub) && rc->callee_qn && + strstr(rc->callee_qn, calleeSub)) + return i; + } + return -1; +} + +TEST(phplsp_cross_reparse_reads_past_inline_html_issue2000) { + const char *src = "\n" + "
    \n" + "

    plain markup after the closing tag

    \n" + "
    \n" + "send(); }\n"; + CBMLSPDef defs[] = { + {.qualified_name = "test.main.gamma", + .short_name = "gamma", + .label = "Function", + .def_module_qn = "test.main"}, + {.qualified_name = "test.main.delta", + .short_name = "delta", + .label = "Function", + .def_module_qn = "test.main"}, + {.qualified_name = "test.mailer.Mailer", + .short_name = "Mailer", + .label = "Class", + .def_module_qn = "test.mailer"}, + {.qualified_name = "test.mailer.Mailer.send", + .short_name = "send", + .label = "Method", + .def_module_qn = "test.mailer", + .receiver_type = "test.mailer.Mailer"}, + }; + const char *imp_names[] = {"Mailer"}; + const char *imp_qns[] = {"test.mailer.Mailer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_php_lsp_cross(&arena, src, (int)strlen(src), "test.main", defs, 4, imp_names, imp_qns, + 1, NULL, &out); + + ASSERT_GTE(find_resolved_arr(&out, "main.delta", "Mailer.send"), 0); + + cbm_arena_destroy(&arena); + PASS(); +} + /* ── 3. Arrow function with typed parameter ───────────────────── */ TEST(phplsp_arrow_function_typed_param) { @@ -5628,6 +5687,7 @@ SUITE(php_lsp) { /* Phase 1 baseline regressions */ RUN_TEST(phplsp_local_method_via_new_assignment); RUN_TEST(phplsp_local_method_via_typed_param); + RUN_TEST(phplsp_cross_reparse_reads_past_inline_html_issue2000); RUN_TEST(phplsp_arrow_function_typed_param); RUN_TEST(phplsp_static_call_resolved); RUN_TEST(phplsp_self_and_parent); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index ceffd1e92..ce6a75741 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -2171,6 +2171,57 @@ TEST(pipeline_nix_scoped_binding_calls_resolve) { PASS(); } +/* #2000, end to end: functions declared after an inline-HTML block reach the + * store with their CALLS edges, in the same file and across files. */ +TEST(pipeline_php_inline_html_calls_resolve_issue2000) { + if (setup_test_repo() != 0) { + FAIL("failed to create temp dir"); + } + + static const struct { + const char *name; + const char *body; + } files[] = { + {"inline_html.php", "\n" + "
    \n" + "

    plain markup after the closing tag

    \n" + "
    \n" + "\n" + "\n" + "

    \n" + " Int { 1 }\n" + "func other() -> Int { 2 }\n" + "\n" + "struct Box {\n" + " let value: Int\n" + " init() {\n" + " value = helper()\n" + " }\n" + " func read() -> Int {\n" + " other()\n" + " }\n" + "}\n" + "\n" + "final class Holder {\n" + " let value: Int\n" + " init(seed: Int) {\n" + " value = helper() + seed\n" + " }\n" + " convenience init() {\n" + " self.init(seed: other())\n" + " }\n" + "}\n" + "\n" + "enum Mode {\n" + " case on\n" + " init?(flag: Bool) {\n" + " if helper() > 0 { self = .on } else { " + "return nil }\n" + " }\n" + "}\n" + "\n" + "extension Box {\n" + " init(raw: Int) {\n" + " value = other()\n" + " }\n" + "}\n"; + +static const char *const SWIFT_INIT_BASE_SRC = "open class Base {\n" + " public init() {}\n" + "}\n"; + +static const char *const SWIFT_INIT_SUB_SRC = "import Kit\n" + "import UIKit\n" + "\n" + "class Sub: Base {\n" + " override init() {\n" + " super.init()\n" + " }\n" + "}\n" + "class Screen: UIViewController {\n" + " init() {\n" + " super.init(nibName: nil, bundle: nil)\n" + " }\n" + " func alert() -> UIAlertAction {\n" + " UIAlertAction.init(title: nil, style: " + ".default)\n" + " }\n" + "}\n" + "extension Notification {\n" + " init(tag: String) {\n" + " self.init(name: Notification.Name(tag))\n" + " }\n" + "}\n" + "func make() -> Box {\n" + " .init(raw: 1)\n" + "}\n" + "func explicit() -> Box {\n" + " Box.init(raw: 2)\n" + "}\n"; + +enum { SWIFT_INIT_EDGE_CAP = 32, SWIFT_INIT_EDGE_LEN = 192 }; + +typedef struct { + int run_rc; + int calls; /* CALLS edges seen; -1 if the store was not read */ + int init_usages; /* USAGE edges into a node named init/deinit */ + char edges[SWIFT_INIT_EDGE_CAP][SWIFT_INIT_EDGE_LEN]; /* "src -> tgt", sorted */ +} SwiftInitObservation; + +static int swift_init_edge_cmp(const void *a, const void *b) { + return strcmp((const char *)a, (const char *)b); +} + +/* Qualified name relative to the project ("Inits.Box.init"). */ +static const char *swift_init_rel_qn(const char *qn, const char *project) { + size_t plen = strlen(project); + return strncmp(qn, project, plen) == 0 && qn[plen] == '.' ? qn + plen + 1 : qn; +} + +static void swift_init_observe(cbm_store_t *s, const char *project, SwiftInitObservation *o) { + cbm_edge_t *edges = NULL; + int count = 0; + o->calls = 0; + if (cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &count) == CBM_STORE_OK) { + for (int i = 0; i < count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && + o->calls < SWIFT_INIT_EDGE_CAP) { + snprintf(o->edges[o->calls], SWIFT_INIT_EDGE_LEN, "%s -> %s", + swift_init_rel_qn(src.qualified_name, project), + swift_init_rel_qn(tgt.qualified_name, project)); + } + o->calls++; + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, count); + } + } + int kept = o->calls < SWIFT_INIT_EDGE_CAP ? o->calls : SWIFT_INIT_EDGE_CAP; + qsort(o->edges, (size_t)kept, sizeof(o->edges[0]), swift_init_edge_cmp); + + edges = NULL; + count = 0; + o->init_usages = -1; /* error sentinel until the USAGE query succeeds */ + if (cbm_store_find_edges_by_type(s, project, "USAGE", &edges, &count) == CBM_STORE_OK) { + o->init_usages = 0; + for (int i = 0; i < count; i++) { + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK && + tgt.name && (strcmp(tgt.name, "init") == 0 || strcmp(tgt.name, "deinit") == 0)) { + o->init_usages++; + } + cbm_node_free_fields(&tgt); + } + if (edges) { + cbm_store_free_edges(edges, count); + } + } +} + +/* `pad` filler files push the run past MIN_FILES_FOR_PARALLEL so that, with + * `parallel`, it goes through pass_parallel.c rather than pass_calls.c. */ +static void swift_init_case(bool parallel, int pad, SwiftInitObservation *o) { + memset(o, 0, sizeof(*o)); + o->run_rc = -1; + o->calls = -1; + o->init_usages = -1; + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_swift_init_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + return; + } + char path[512]; + bool written = true; + const char *names[] = {"Package.swift", "Sources/Kit/Inits.swift", "Sources/Kit/Base.swift", + "Sources/App/Sub.swift"}; + const char *sources[] = {SWIFT_INIT_PACKAGE_SRC, SWIFT_INIT_INITS_SRC, SWIFT_INIT_BASE_SRC, + SWIFT_INIT_SUB_SRC}; + for (int i = 0; i < 4 && written; i++) { + snprintf(path, sizeof(path), "%s/%s", tmp, names[i]); + written = th_write_file(path, sources[i]) == 0; + } + for (int i = 0; i < pad && written; i++) { + char source[96]; + snprintf(path, sizeof(path), "%s/Pad%02d.swift", tmp, i); + snprintf(source, sizeof(source), "func pad%02d() -> Int { %d }\n", i, i); + written = th_write_file(path, source) == 0; + } + if (written) { + XlangEnvSave save = xlang_env_select(parallel); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/swift_init.db", tmp); + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + if (pipeline) { + o->run_rc = cbm_pipeline_run(pipeline); + const char *project = cbm_pipeline_project_name(pipeline); + cbm_store_t *store = cbm_store_open_path(db_path); + if (store && project) { + swift_init_observe(store, project, o); + } + if (store) { + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + } + xlang_env_restore(&save); + } + th_rmtree(tmp); +} + +/* The exact CALLS set. Every initializer's own calls come from that + * initializer (the repro's file-level collapse is gone); Sub.init reaches + * Base.init in the other target and `Box.init(raw:)` reaches Box.init. + * UIViewController's and UIAlertAction's initializers, Notification's own and + * the implicit `.init(...)` are outside the project and form no edge -- in + * particular none to a project `init` that merely shares the name. Holder's + * convenience `self.init` is not a self-loop. */ +static int swift_init_assert_exact(const SwiftInitObservation *o) { + static const char *const expected[] = { + "Sources.App.Sub.Sub.init -> Sources.Kit.Base.Base.init", + "Sources.App.Sub.explicit -> Sources.Kit.Inits.Box.init", + "Sources.Kit.Inits.Box.init -> Sources.Kit.Inits.helper", + "Sources.Kit.Inits.Box.init -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Box.read -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Holder.init -> Sources.Kit.Inits.helper", + "Sources.Kit.Inits.Holder.init -> Sources.Kit.Inits.other", + "Sources.Kit.Inits.Mode.init -> Sources.Kit.Inits.helper", + }; + enum { EXPECTED_CALLS = sizeof(expected) / sizeof(expected[0]) }; + ASSERT_EQ(o->run_rc, 0); + ASSERT_EQ(o->init_usages, 0); + ASSERT_EQ(o->calls, (int)EXPECTED_CALLS); + for (int i = 0; i < (int)EXPECTED_CALLS; i++) { + ASSERT_STR_EQ(o->edges[i], expected[i]); + } + return 0; +} + +TEST(pipeline_swift_init_exact_calls_sequential_issue2379) { + SwiftInitObservation o; + swift_init_case(false, 0, &o); + if (swift_init_assert_exact(&o) != 0) { + return 1; + } + PASS(); +} + +/* Same input through pass_parallel.c: 55 fillers exceed MIN_FILES_FOR_PARALLEL. */ +TEST(pipeline_swift_init_exact_calls_parallel_issue2379) { + SwiftInitObservation o; + swift_init_case(true, 55, &o); + if (swift_init_assert_exact(&o) != 0) { + return 1; + } + PASS(); +} + TEST(pipeline_python_cross_module_call) { /* Port of TestPythonCrossModuleCallViaImport */ const char *files[] = {"utils.py", "main.py"}; @@ -27032,6 +27332,72 @@ TEST(cancelled_incremental_reindex_preserves_committed_db) { PASS(); } +typedef struct { + bool allow; + int calls; +} publish_gate_ctx_t; + +static bool publish_gate_decide(void *context) { + publish_gate_ctx_t *gate = context; + gate->calls++; + return gate->allow; +} + +/* The publication gate (a supervised worker's claim on its publication fence) + * is the last word before the serving database is touched: refused, the run + * stops with the previous generation's file in place; allowed, it publishes + * as before. Asked exactly once either way. */ +TEST(publish_gate_decides_before_the_committed_db_is_touched) { + if (setup_incremental_repo() != 0) { + FAIL("setup failed"); + } + cbm_pipeline_t *p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + char *project = strdup(cbm_pipeline_project_name(p)); + cbm_pipeline_free(p); + ASSERT_NOT_NULL(project); + struct stat baseline; + ASSERT_EQ(stat(g_incr_dbpath, &baseline), 0); + + ASSERT_TRUE(write_go_file(g_incr_tmpdir, "gate_candidate.go", + "package main\n\nfunc GateCandidate() int { return 7 }\n")); + publish_gate_ctx_t refused = {.allow = false}; + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_publish_gate(p, publish_gate_decide, &refused); + int refused_rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + struct stat after_refusal; + bool same_file = + stat(g_incr_dbpath, &after_refusal) == 0 && after_refusal.st_ino == baseline.st_ino; + cbm_store_t *live = cbm_store_open_path(g_incr_dbpath); + int refused_candidates = live ? count_nodes_named(live, project, "GateCandidate") : -1; + cbm_store_close(live); + + publish_gate_ctx_t allowed = {.allow = true}; + p = cbm_pipeline_new(g_incr_tmpdir, g_incr_dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + cbm_pipeline_set_publish_gate(p, publish_gate_decide, &allowed); + int allowed_rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + live = cbm_store_open_path(g_incr_dbpath); + int allowed_candidates = live ? count_nodes_named(live, project, "GateCandidate") : -1; + cbm_store_close(live); + + free(project); + cleanup_incremental_repo(); + + ASSERT_EQ(refused.calls, 1); + ASSERT_EQ(refused_rc, CBM_PIPELINE_ABORT_PRESERVE_DB); + ASSERT_TRUE(same_file); + ASSERT_EQ(refused_candidates, 0); + ASSERT_EQ(allowed.calls, 1); + ASSERT_EQ(allowed_rc, 0); + ASSERT_GT(allowed_candidates, 0); + PASS(); +} + TEST(backup_failed_publish_failure_preserves_final_sidecars) { if (setup_incremental_repo() != 0) { FAIL("setup failed"); @@ -31582,6 +31948,7 @@ SUITE(pipeline) { /* Calls pass */ RUN_TEST(pipeline_calls_resolution); RUN_TEST(pipeline_nix_scoped_binding_calls_resolve); + RUN_TEST(pipeline_php_inline_html_calls_resolve_issue2000); RUN_TEST(pipeline_hcl_block_reference_resolves_to_its_block); RUN_TEST(pipeline_incremental_preserves_cross_file_calls); RUN_TEST(pipeline_incremental_drops_terraform_reads_of_a_moved_module); @@ -31762,6 +32129,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_imports_multi_symbol_edges); RUN_TEST(pipeline_go_cross_package_call); RUN_TEST(pipeline_swift_cross_package_import); + RUN_TEST(pipeline_swift_init_exact_calls_sequential_issue2379); + RUN_TEST(pipeline_swift_init_exact_calls_parallel_issue2379); RUN_TEST(pipeline_python_cross_module_call); RUN_TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725); RUN_TEST(pipeline_go_type_classification); @@ -31893,6 +32262,7 @@ SUITE(pipeline) { RUN_TEST(incremental_new_file_added); RUN_TEST(cancelled_full_reindex_preserves_committed_db); RUN_TEST(cancelled_incremental_reindex_preserves_committed_db); + RUN_TEST(publish_gate_decides_before_the_committed_db_is_touched); RUN_TEST(backup_failed_publish_failure_preserves_final_sidecars); RUN_TEST(backup_failed_rename_failure_preserves_corrupt_main); #ifdef __linux__ diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index 90ee2ab4c..2ae369e85 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -101,6 +101,12 @@ TEST(subprocess_outcome_str) { PASS(); } +TEST(subprocess_tree_rss_sum_saturates_on_overflow) { + const uint64_t values[] = {UINT64_MAX - 4, 4, 1}; + ASSERT_EQ(cbm_subprocess_rss_sum_for_testing(values, 3), UINT64_MAX); + PASS(); +} + /* ── Layer 2: real spawn/reap (POSIX) ─────────────────────────────────────── */ extern const char *cbm_test_program_path; @@ -807,6 +813,190 @@ TEST(subprocess_spawn_returns_while_child_is_running) { #endif } +/* ── Process-tree memory probe (every platform) ───────────────────────────── + * The test runner's __cbm_rss_tree_root role starts one __cbm_rss_tree_hold + * descendant that touches a controlled allocation and then publishes its pid. + * Waits below have no deadline: they end when the descendant is ready or the + * tree stops running, so the outcome, not a timer, decides the verdict. */ + +#ifdef _WIN32 +#define RSS_TREE_FAULT_GONE ERROR_INVALID_PARAMETER /* no process with that id any more */ +#define RSS_TREE_FAULT_LIVE ERROR_ACCESS_DENIED /* a member we may not open */ +#elif defined(__APPLE__) +#define RSS_TREE_FAULT_GONE ESRCH +#define RSS_TREE_FAULT_LIVE EPERM +#else +#define RSS_TREE_FAULT_GONE ENOENT +#define RSS_TREE_FAULT_LIVE EACCES +#endif + +#define RSS_TREE_MIB UINT64_C(1048576) + +typedef struct { + bool spawned; + bool ready; + cbm_proc_tree_rss_status_t status; + uint64_t rss; + bool quiesced; + cbm_proc_tree_rss_status_t final_status; + uint64_t final_rss; +} rss_tree_sample_t; + +static void rss_tree_pause(void) { + cbm_usleep(10000); /* poll interval only; nothing here is a deadline */ +} + +static bool rss_tree_wait_ready(const char *ready_path, cbm_subprocess_t *process) { + for (;;) { + FILE *ready = cbm_fopen(ready_path, "rb"); + if (ready) { + long pid = 0; + int fields = fscanf(ready, "%ld", &pid); + (void)fclose(ready); + if (fields == 1 && pid > 0) { + return true; + } + } + cbm_proc_result_t ignored; + if (cbm_subprocess_poll(process, &ignored) != CBM_PROC_POLL_RUNNING) { + return false; + } + rss_tree_pause(); + } +} + +static bool rss_tree_finish(cbm_subprocess_t *process, rss_tree_sample_t *sample) { + (void)cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + for (;;) { + cbm_proc_poll_t state = cbm_subprocess_poll(process, &result); + if (state == CBM_PROC_POLL_TERMINAL) { + break; + } + if (state == CBM_PROC_POLL_ERROR) { + return false; + } + rss_tree_pause(); + } + sample->final_status = cbm_subprocess_tree_rss_bytes(process, &sample->final_rss); + cbm_subprocess_destroy(process); + return result.tree_quiesced; +} + +/* Measure one tree whose descendant holds `mib` MiB, with every non-root + * member's measurement failing with `member_fault` (0 = real calls). */ +static rss_tree_sample_t rss_tree_sample_in(int mib, int member_fault, bool own_group) { + rss_tree_sample_t sample = {.status = CBM_PROC_TREE_RSS_ERROR, + .final_status = CBM_PROC_TREE_RSS_ERROR, + .final_rss = UINT64_MAX}; + const char *made = th_mktempdir("cbm_rss_tree"); + char dir[256]; + char ready_path[300]; + char mib_text[16]; + if (!made || snprintf(dir, sizeof(dir), "%s", made) >= (int)sizeof(dir) || + snprintf(ready_path, sizeof(ready_path), "%s/ready", dir) >= (int)sizeof(ready_path)) { + return sample; + } + (void)snprintf(mib_text, sizeof(mib_text), "%d", mib); + const char *argv[] = {cbm_test_program_path, + "__cbm_rss_tree_root", + ready_path, + mib_text, + own_group ? "detach" : NULL, + NULL}; + cbm_proc_opts_t opts = {0}; + opts.bin = cbm_test_program_path; + opts.argv = argv; + opts.cancel_grace_ms = 100; + cbm_subprocess_t *process = NULL; + sample.spawned = cbm_subprocess_spawn(&opts, &process) == 0; + if (sample.spawned) { + sample.ready = rss_tree_wait_ready(ready_path, process); + if (sample.ready) { + cbm_subprocess_tree_rss_fault_members_for_testing(member_fault); + sample.status = cbm_subprocess_tree_rss_bytes(process, &sample.rss); + cbm_subprocess_tree_rss_fault_members_for_testing(0); + } + sample.quiesced = rss_tree_finish(process, &sample); + } + th_cleanup(dir); + return sample; +} + +static rss_tree_sample_t rss_tree_sample(int mib, int member_fault) { + return rss_tree_sample_in(mib, member_fault, false); +} + +/* Only the DESCENDANT allocates, so the difference between two otherwise + * identical trees is memory the probe can only have counted by measuring past + * the root. 128 MiB touched against a 96 MiB floor leaves room for allocator + * and page-rounding noise and none for a root-only total. */ +TEST(subprocess_tree_rss_measures_contained_descendants) { + rss_tree_sample_t idle = rss_tree_sample(0, 0); + rss_tree_sample_t loaded = rss_tree_sample(128, 0); + if (loaded.status == CBM_PROC_TREE_RSS_OK && idle.status == CBM_PROC_TREE_RSS_OK) { + printf(" tree rss: idle=%llu MiB loaded=%llu MiB\n", + (unsigned long long)(idle.rss / RSS_TREE_MIB), + (unsigned long long)(loaded.rss / RSS_TREE_MIB)); + } + ASSERT_TRUE(idle.spawned); + ASSERT_TRUE(idle.ready); + ASSERT_EQ(idle.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(idle.rss > 0); + ASSERT_TRUE(loaded.spawned); + ASSERT_TRUE(loaded.ready); + ASSERT_EQ(loaded.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.rss >= idle.rss + 96 * RSS_TREE_MIB); + ASSERT_TRUE(idle.quiesced); + ASSERT_TRUE(loaded.quiesced); + ASSERT_EQ(loaded.final_status, CBM_PROC_TREE_RSS_EMPTY); + ASSERT_EQ(loaded.final_rss, 0); + PASS(); +} + +/* git discovery children run in a process group of their own. The probe must + * still count them: the memory is the worker's, and a group-only probe would + * let it grow past the limit unseen. POSIX only (a Job Object has no groups). */ +TEST(subprocess_tree_rss_counts_descendants_in_their_own_process_group) { +#ifdef _WIN32 + SKIP_PLATFORM("process groups are POSIX; the Job Object probe already covers descendants"); +#else + rss_tree_sample_t idle = rss_tree_sample_in(0, 0, true); + rss_tree_sample_t loaded = rss_tree_sample_in(128, 0, true); + ASSERT_TRUE(idle.ready); + ASSERT_EQ(idle.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.ready); + ASSERT_EQ(loaded.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(loaded.rss >= idle.rss + 96 * RSS_TREE_MIB); + ASSERT_TRUE(idle.quiesced); + ASSERT_TRUE(loaded.quiesced); + PASS(); +#endif +} + +/* A live member the platform will not let us measure must fail the probe: a + * total without it would under-report the tree. */ +TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured) { + rss_tree_sample_t sample = rss_tree_sample(0, RSS_TREE_FAULT_LIVE); + ASSERT_TRUE(sample.spawned); + ASSERT_TRUE(sample.ready); + ASSERT_EQ(sample.status, CBM_PROC_TREE_RSS_ERROR); + ASSERT_TRUE(sample.quiesced); + PASS(); +} + +/* A member that exited between the listing and its measurement holds nothing: + * it is skipped and the rest of the tree (here the root) is still measured. */ +TEST(subprocess_tree_rss_skips_a_member_that_exited_mid_probe) { + rss_tree_sample_t sample = rss_tree_sample(0, RSS_TREE_FAULT_GONE); + ASSERT_TRUE(sample.spawned); + ASSERT_TRUE(sample.ready); + ASSERT_EQ(sample.status, CBM_PROC_TREE_RSS_OK); + ASSERT_TRUE(sample.rss > 0); + ASSERT_TRUE(sample.quiesced); + PASS(); +} + TEST(subprocess_natural_completion_is_cached_across_polls) { #ifdef _WIN32 SKIP_PLATFORM("POSIX /bin/sh completion-cache probe; native Job Object coverage pending"); @@ -2641,6 +2831,7 @@ SUITE(subprocess) { #ifndef _WIN32 RUN_TEST(subprocess_environment_path_preserves_caller_lookup); #endif + RUN_TEST(subprocess_tree_rss_sum_saturates_on_overflow); RUN_TEST(subprocess_run_clean); RUN_TEST(subprocess_run_exit_nonzero); RUN_TEST(subprocess_run_child_exited_before_group_check); @@ -2654,6 +2845,10 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_spawn_failure); RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); + RUN_TEST(subprocess_tree_rss_measures_contained_descendants); + RUN_TEST(subprocess_tree_rss_counts_descendants_in_their_own_process_group); + RUN_TEST(subprocess_tree_rss_fails_when_a_live_member_cannot_be_measured); + RUN_TEST(subprocess_tree_rss_skips_a_member_that_exited_mid_probe); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); diff --git a/tests/test_worker_session_scope.sh b/tests/test_worker_session_scope.sh index e6a94a3f8..3aff3a6cb 100644 --- a/tests/test_worker_session_scope.sh +++ b/tests/test_worker_session_scope.sh @@ -73,9 +73,9 @@ run_worker() { # A worker never reads the resource policy from config or environment: the # supervisor resolves it and sends it inside the request, and a request # without one is refused ("missing or incomplete trusted worker policy"). -# This script plays the supervisor, so it sends what the supervisor sends -- -# both limits off, the default. -policy='"_cbm_index_policy":{"index_max_files":"off","index_max_source_mb":"off"}' +# This script plays the supervisor, so it sends the same complete object +# cbm_test_index_worker_policy_json produces -- every known key, all off. +policy="$(cbm_test_index_worker_policy_json)" response="${tmpdir}/scoped.response" if ! run_worker "{\"repo_path\":\"${repo}\",\"mode\":\"fast\",${policy}}" "${response}" \