Upstream sync 13M-2: merge DeusData e0c7a1da6 with #1724 resource worker port

.github/workflows/_build.yml

@@ -323,7 +323,7 @@ jobs:
           persist-credentials: false
 
       - name: Download all native candidate pairs
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           pattern: release-candidates-*
           path: ${{ runner.temp }}/release-candidate-artifacts

.github/workflows/_smoke.yml

@@ -111,7 +111,14 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      # SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper; the
+      # macOS runner images have none on PATH.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: pkg/go/go.mod
+          cache: false
+
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
 
@@ -211,7 +218,7 @@ jobs:
             zip
             coreutils
 
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-windows-${{ matrix.arch }}
 
@@ -304,7 +311,13 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      # SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: pkg/go/go.mod
+          cache: false
+
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-linux-${{ matrix.arch }}-portable
 

.github/workflows/_soak.yml

@@ -72,7 +72,7 @@ jobs:
         run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
       - name: Download selected release container
         if: ${{ inputs.use_release_artifacts }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
           path: ${{ runner.temp }}/cbm-selected-container
@@ -147,7 +147,7 @@ jobs:
         run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
       - name: Download selected release container
         if: ${{ inputs.use_release_artifacts }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-windows-amd64
           path: ${{ runner.temp }}/cbm-selected-container
@@ -226,7 +226,7 @@ jobs:
         run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
       - name: Download selected release container
         if: ${{ inputs.use_release_artifacts }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-windows-arm64
           path: ${{ runner.temp }}/cbm-selected-container
@@ -288,7 +288,7 @@ jobs:
         timeout-minutes: 10
         run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y python3 git
       - name: Download selected portable release container
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: binaries-linux-${{ matrix.arch }}-portable
           path: ${{ runner.temp }}/cbm-selected-container

.github/workflows/_test.yml

@@ -246,7 +246,7 @@ jobs:
         # producer, so arm64, macOS and Windows code under #ifdef has static
         # evidence only and those legs run every test.
         if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: ${{ inputs.test_selection }}
           path: ${{ runner.temp }}/test-selection
@@ -746,7 +746,7 @@ jobs:
         with:
           path: repo
 
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           pattern: shard-manifest-*
           path: manifests

.github/workflows/release.yml

@@ -231,7 +231,7 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Download only selected release containers
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           pattern: binaries-*
           merge-multiple: true
@@ -240,7 +240,7 @@ jobs:
       # download above. Preserve only the three reviewable manifests that bind
       # candidate bytes, VT results, and the tuple-local decision.
       - name: Download candidate selection evidence separately
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: release-selection-evidence
           path: selection-evidence

Makefile.cbm

@@ -419,6 +419,7 @@ EXTRACTION_SRCS = \
     $(CBM_DIR)/openapi.c \
     $(CBM_DIR)/extract_nextjs.c \
     $(CBM_DIR)/sql_values.c \
+    $(CBM_DIR)/php_inline_html.c \
     $(CBM_DIR)/helpers.c \
     $(CBM_DIR)/cpp_branch_views.c \
     $(CBM_DIR)/result_compact.c \

docs/CONFIGURATION.md

@@ -101,6 +101,8 @@ Current keys:
 | `watcher_enabled` | `true` | Master switch for the retained background watcher subsystem. Set `false` to stop its poll thread/project registration. Reindex manually with `codebase-memory-cli index` when disabled. |
 | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
 | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
+| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. |
+| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). Like `index_max_rss_mb`, it stops a worker only before it starts to publish the new index; see [INDEX_RESOURCE_LIMITS.md](INDEX_RESOURCE_LIMITS.md). |
 
 > **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
 > retained watcher subsystem starts at all. `auto_watch` is narrower: it controls
@@ -112,11 +114,13 @@ Current keys:
 > value. Disabling the watcher does not disable explicit `index`, `search`,
 > `trace`, `snippet`, or `coverage` commands.
 
-The two `index_max_*` settings are independent and disabled by default. They
+The four `index_max_*` settings are independent and disabled by default. They
 apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
 to `cross-repo-intelligence`, which does not scan repository source files.
-Equality is allowed; exceeding either setting fails the complete index request
-and preserves any previously serving database. See
+Equality is allowed; exceeding any setting fails the complete index request and
+preserves any previously serving database. Worker RSS covers descendants and is
+not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total
+duration is independent of the existing 15-minute no-log-progress timeout. See
 [Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
 error-response details.
 

docs/INDEX_RESOURCE_LIMITS.md

@@ -1,8 +1,8 @@
 # Index resource limits
 
 Index resource limits are optional operator controls for repositories whose
-discovery breadth is not known in advance. They are disabled by default so
-existing large-repository workloads retain their current behavior.
+discovery breadth or worker runtime is not known in advance. They are disabled
+by default so existing large-repository workloads retain their current behavior.
 
 ## Discovery settings
 
@@ -23,6 +23,61 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
 negative values, suffixes, trailing characters, and values outside the stated
 ranges are rejected without changing the stored value.
 
+## Worker settings
+
+| Key | Default | Accepted value | Protects |
+|---|---:|---:|---|
+| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree |
+| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request |
+
+```bash
+codebase-memory-cli config set index_max_rss_mb 8192
+codebase-memory-cli config set index_max_duration_seconds 3600
+```
+
+RSS is the charged memory of the contained worker and every descendant, not
+the worker's allocation budget and not peak memory. On macOS that quantity is
+`phys_footprint` (the same number `cbm_mem_charged()` enforces), not
+`resident_size`, which still counts pages the allocator has already handed
+back. Linux and Windows use RSS / working set. This hard watchdog is
+separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor
+samples the tree at most once every 250 milliseconds so the watchdog does not
+turn full process-table enumeration into a busy loop.
+
+Duration is per request: the clock starts at the first worker spawn and is
+not reset when crash/hang recovery starts a later attempt. Continuous log
+progress does not reset it. It is independent of the existing 15-minute quiet
+timeout, which still identifies a worker that stops making progress. A
+duration limit shorter than that quiet timeout kills a hung worker before hang
+quarantine can name the in-flight file, so the next attempt may hang on the
+same file.
+
+Equality is allowed. The first RSS or elapsed-duration observation above its
+limit starts the existing graceful-to-force process-tree shutdown. CBM reports
+terminal only after the tree is quiescent or a bounded containment failure is
+explicitly surfaced. Resource termination is not retried and does not
+quarantine a source file.
+
+A worker limit never stops a worker after it has started to publish. The
+supervisor enforces the limits, but only the worker replaces the serving
+database, so the two coordinate through a per-attempt publication claim that
+exactly one of them can win. The worker claims it immediately before it
+replaces the serving database; the supervisor claims it before it stops the
+worker for a limit. If the limit is reached first, the worker can no longer
+publish and the previous index stays untouched. If the worker claimed first,
+the limits no longer apply to that attempt: the new index is published and the
+rest of the attempt, including the persistence artifact export, runs to
+completion. The supervisor then logs `index.supervisor.resource_limit_after_publication`
+instead of stopping it. With both worker limits `off` there is no claim.
+
+If RSS is enabled and three consecutive probes cannot obtain any trustworthy
+tree measurement while the root worker is still running, CBM fails closed with
+`code=resource_probe_failed`. A probe is untrustworthy as soon as one member of
+the tree cannot be measured: only a process that exits between the listing and
+its measurement is skipped, because a total without a live member would
+under-report. On Linux the probe reads every `/proc/<pid>/stat` to find the
+tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed.
+
 ## Counting and failure semantics
 
 `index_max_files` counts a file only after it passes directory pruning, ignore
@@ -55,6 +110,15 @@ The previous database remains available because publication occurs only after a
 complete discovery and successful staged build. If no previous database exists,
 `serving_index_preserved` is false.
 
+Worker limit failures use the same shape with `stage=worker`,
+`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and
+`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed`
+and omit `observed`, `limit`, and `unit` because no trustworthy observation was
+available. For a worker failure `serving_index_preserved` describes the whole
+request: it is true only when a servable index existed when the request began,
+no attempt of the request (including an earlier one that crashed and was
+recovered) claimed publication, and that index still serves.
+
 ## Trust and compatibility
 
 Limits are read from the CLI-managed `_config.db`; they are not index request
@@ -64,5 +128,6 @@ parent policy.
 
 These settings do not replace or increase `auto_index_limit`, change the 512 MiB
 single-file cap, alter workspace-root authorization, or affect
-`cross-repo-intelligence`. With both settings `off`, discovery follows the
-existing unbounded path.
+`cross-repo-intelligence`. With all settings `off`, discovery follows the
+existing path and the supervisor performs no periodic RSS probe or total-duration
+termination.

internal/cbm/cbm.c

@@ -3321,6 +3321,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
     return r;
 }
 
+/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
+ * is parsed with its markup blanked (#2000); the rewrite keeps the length and
+ * every line break, so offsets and line numbers still refer to the file on
+ * disk. Every other language parses the file as it is. */
+static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
+                                       int source_len) {
+    if (language == CBM_LANG_PHP) {
+        return cbm_php_mask_inline_html(a, source, source_len);
+    }
+    return source;
+}
+
 /* Initial block for the per-file traversal scratch arena, chosen by measuring
  * arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
  * both this size and at 1 MB, and on most files at 256 KB, where the two
@@ -3741,6 +3753,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
         return result;
     }
 
+    source = extract_parse_bytes(a, language, source, source_len);
+
     // Skip pathologically nested Perl before tree-sitter's recursive GLR stack
     // merge overflows a small stack during the parse (see
     // cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is

internal/cbm/cbm.h

@@ -1385,6 +1385,10 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx);
 // per ref()/source() call. No-op unless the file parses as SQL and actually
 // contains a dbt builtin call. Defined in extract_dbt.c.
 void cbm_extract_dbt(CBMExtractCtx *ctx);
+// PHP source with its inline-HTML regions blanked so the php_only grammar can
+// parse past `?>` (#2000). Same length and line breaks as the input; returns
+// `source` itself when nothing needs rewriting. Defined in php_inline_html.c.
+const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len);
 void cbm_extract_imports(CBMExtractCtx *ctx);
 void cbm_extract_usages(CBMExtractCtx *ctx);
 void cbm_extract_semantic(CBMExtractCtx *ctx);