Skip to content

Commit 89e57fa

Browse files
authored
Merge pull request #34 from ngallodev/sync/upstream-13m2-20261108
Upstream sync 13M-2: merge DeusData e0c7a1d with DeusData#1724 resource worker port
2 parents 929adfe + f2b0dae commit 89e57fa

55 files changed

Lines changed: 4953 additions & 173 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ jobs:
323323
persist-credentials: false
324324

325325
- name: Download all native candidate pairs
326-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
326+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
327327
with:
328328
pattern: release-candidates-*
329329
path: ${{ runner.temp }}/release-candidate-artifacts

‎.github/workflows/_smoke.yml‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,14 @@ jobs:
111111
steps:
112112
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
113113

114-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
114+
# SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper; the
115+
# macOS runner images have none on PATH.
116+
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
117+
with:
118+
go-version-file: pkg/go/go.mod
119+
cache: false
120+
121+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
115122
with:
116123
name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
117124

@@ -211,7 +218,7 @@ jobs:
211218
zip
212219
coreutils
213220
214-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
221+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
215222
with:
216223
name: binaries-windows-${{ matrix.arch }}
217224

@@ -304,7 +311,13 @@ jobs:
304311
steps:
305312
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
306313

307-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
314+
# SMOKE_REQUIRE_WRAPPERS=1 needs a Go toolchain for the Go wrapper.
315+
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
316+
with:
317+
go-version-file: pkg/go/go.mod
318+
cache: false
319+
320+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
308321
with:
309322
name: binaries-linux-${{ matrix.arch }}-portable
310323

‎.github/workflows/_soak.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ jobs:
7272
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
7373
- name: Download selected release container
7474
if: ${{ inputs.use_release_artifacts }}
75-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
75+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
7676
with:
7777
name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
7878
path: ${{ runner.temp }}/cbm-selected-container
@@ -147,7 +147,7 @@ jobs:
147147
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
148148
- name: Download selected release container
149149
if: ${{ inputs.use_release_artifacts }}
150-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
150+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
151151
with:
152152
name: binaries-windows-amd64
153153
path: ${{ runner.temp }}/cbm-selected-container
@@ -226,7 +226,7 @@ jobs:
226226
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
227227
- name: Download selected release container
228228
if: ${{ inputs.use_release_artifacts }}
229-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
229+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
230230
with:
231231
name: binaries-windows-arm64
232232
path: ${{ runner.temp }}/cbm-selected-container
@@ -288,7 +288,7 @@ jobs:
288288
timeout-minutes: 10
289289
run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y python3 git
290290
- name: Download selected portable release container
291-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
291+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
292292
with:
293293
name: binaries-linux-${{ matrix.arch }}-portable
294294
path: ${{ runner.temp }}/cbm-selected-container

‎.github/workflows/_test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ jobs:
246246
# producer, so arm64, macOS and Windows code under #ifdef has static
247247
# evidence only and those legs run every test.
248248
if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }}
249-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
249+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
250250
with:
251251
name: ${{ inputs.test_selection }}
252252
path: ${{ runner.temp }}/test-selection
@@ -746,7 +746,7 @@ jobs:
746746
with:
747747
path: repo
748748

749-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
749+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
750750
with:
751751
pattern: shard-manifest-*
752752
path: manifests

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ jobs:
231231
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
232232

233233
- name: Download only selected release containers
234-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
234+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
235235
with:
236236
pattern: binaries-*
237237
merge-multiple: true
@@ -240,7 +240,7 @@ jobs:
240240
# download above. Preserve only the three reviewable manifests that bind
241241
# candidate bytes, VT results, and the tuple-local decision.
242242
- name: Download candidate selection evidence separately
243-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
243+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
244244
with:
245245
name: release-selection-evidence
246246
path: selection-evidence

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ EXTRACTION_SRCS = \
419419
$(CBM_DIR)/openapi.c \
420420
$(CBM_DIR)/extract_nextjs.c \
421421
$(CBM_DIR)/sql_values.c \
422+
$(CBM_DIR)/php_inline_html.c \
422423
$(CBM_DIR)/helpers.c \
423424
$(CBM_DIR)/cpp_branch_views.c \
424425
$(CBM_DIR)/result_compact.c \

‎docs/CONFIGURATION.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ Current keys:
101101
| `watcher_enabled` | `true` | Master switch for the retained background watcher subsystem. Set `false` to stop its poll thread/project registration. Reindex manually with `codebase-memory-cli index` when disabled. |
102102
| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
103103
| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
104+
| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. |
105+
| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). Like `index_max_rss_mb`, it stops a worker only before it starts to publish the new index; see [INDEX_RESOURCE_LIMITS.md](INDEX_RESOURCE_LIMITS.md). |
104106

105107
> **`watcher_enabled` vs `auto_watch`.** `watcher_enabled` controls whether the
106108
> retained watcher subsystem starts at all. `auto_watch` is narrower: it controls
@@ -112,11 +114,13 @@ Current keys:
112114
> value. Disabling the watcher does not disable explicit `index`, `search`,
113115
> `trace`, `snippet`, or `coverage` commands.
114116
115-
The two `index_max_*` settings are independent and disabled by default. They
117+
The four `index_max_*` settings are independent and disabled by default. They
116118
apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
117119
to `cross-repo-intelligence`, which does not scan repository source files.
118-
Equality is allowed; exceeding either setting fails the complete index request
119-
and preserves any previously serving database. See
120+
Equality is allowed; exceeding any setting fails the complete index request and
121+
preserves any previously serving database. Worker RSS covers descendants and is
122+
not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total
123+
duration is independent of the existing 15-minute no-log-progress timeout. See
120124
[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
121125
error-response details.
122126

‎docs/INDEX_RESOURCE_LIMITS.md‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Index resource limits
22

33
Index resource limits are optional operator controls for repositories whose
4-
discovery breadth is not known in advance. They are disabled by default so
5-
existing large-repository workloads retain their current behavior.
4+
discovery breadth or worker runtime is not known in advance. They are disabled
5+
by default so existing large-repository workloads retain their current behavior.
66

77
## Discovery settings
88

@@ -23,6 +23,61 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
2323
negative values, suffixes, trailing characters, and values outside the stated
2424
ranges are rejected without changing the stored value.
2525

26+
## Worker settings
27+
28+
| Key | Default | Accepted value | Protects |
29+
|---|---:|---:|---|
30+
| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree |
31+
| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request |
32+
33+
```bash
34+
codebase-memory-cli config set index_max_rss_mb 8192
35+
codebase-memory-cli config set index_max_duration_seconds 3600
36+
```
37+
38+
RSS is the charged memory of the contained worker and every descendant, not
39+
the worker's allocation budget and not peak memory. On macOS that quantity is
40+
`phys_footprint` (the same number `cbm_mem_charged()` enforces), not
41+
`resident_size`, which still counts pages the allocator has already handed
42+
back. Linux and Windows use RSS / working set. This hard watchdog is
43+
separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor
44+
samples the tree at most once every 250 milliseconds so the watchdog does not
45+
turn full process-table enumeration into a busy loop.
46+
47+
Duration is per request: the clock starts at the first worker spawn and is
48+
not reset when crash/hang recovery starts a later attempt. Continuous log
49+
progress does not reset it. It is independent of the existing 15-minute quiet
50+
timeout, which still identifies a worker that stops making progress. A
51+
duration limit shorter than that quiet timeout kills a hung worker before hang
52+
quarantine can name the in-flight file, so the next attempt may hang on the
53+
same file.
54+
55+
Equality is allowed. The first RSS or elapsed-duration observation above its
56+
limit starts the existing graceful-to-force process-tree shutdown. CBM reports
57+
terminal only after the tree is quiescent or a bounded containment failure is
58+
explicitly surfaced. Resource termination is not retried and does not
59+
quarantine a source file.
60+
61+
A worker limit never stops a worker after it has started to publish. The
62+
supervisor enforces the limits, but only the worker replaces the serving
63+
database, so the two coordinate through a per-attempt publication claim that
64+
exactly one of them can win. The worker claims it immediately before it
65+
replaces the serving database; the supervisor claims it before it stops the
66+
worker for a limit. If the limit is reached first, the worker can no longer
67+
publish and the previous index stays untouched. If the worker claimed first,
68+
the limits no longer apply to that attempt: the new index is published and the
69+
rest of the attempt, including the persistence artifact export, runs to
70+
completion. The supervisor then logs `index.supervisor.resource_limit_after_publication`
71+
instead of stopping it. With both worker limits `off` there is no claim.
72+
73+
If RSS is enabled and three consecutive probes cannot obtain any trustworthy
74+
tree measurement while the root worker is still running, CBM fails closed with
75+
`code=resource_probe_failed`. A probe is untrustworthy as soon as one member of
76+
the tree cannot be measured: only a process that exits between the listing and
77+
its measurement is skipped, because a total without a live member would
78+
under-report. On Linux the probe reads every `/proc/<pid>/stat` to find the
79+
tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed.
80+
2681
## Counting and failure semantics
2782

2883
`index_max_files` counts a file only after it passes directory pruning, ignore
@@ -55,6 +110,15 @@ The previous database remains available because publication occurs only after a
55110
complete discovery and successful staged build. If no previous database exists,
56111
`serving_index_preserved` is false.
57112

113+
Worker limit failures use the same shape with `stage=worker`,
114+
`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and
115+
`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed`
116+
and omit `observed`, `limit`, and `unit` because no trustworthy observation was
117+
available. For a worker failure `serving_index_preserved` describes the whole
118+
request: it is true only when a servable index existed when the request began,
119+
no attempt of the request (including an earlier one that crashed and was
120+
recovered) claimed publication, and that index still serves.
121+
58122
## Trust and compatibility
59123

60124
Limits are read from the CLI-managed `_config.db`; they are not index request
@@ -64,5 +128,6 @@ parent policy.
64128

65129
These settings do not replace or increase `auto_index_limit`, change the 512 MiB
66130
single-file cap, alter workspace-root authorization, or affect
67-
`cross-repo-intelligence`. With both settings `off`, discovery follows the
68-
existing unbounded path.
131+
`cross-repo-intelligence`. With all settings `off`, discovery follows the
132+
existing path and the supervisor performs no periodic RSS probe or total-duration
133+
termination.

‎internal/cbm/cbm.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3321,6 +3321,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
33213321
return r;
33223322
}
33233323

3324+
/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
3325+
* is parsed with its markup blanked (#2000); the rewrite keeps the length and
3326+
* every line break, so offsets and line numbers still refer to the file on
3327+
* disk. Every other language parses the file as it is. */
3328+
static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
3329+
int source_len) {
3330+
if (language == CBM_LANG_PHP) {
3331+
return cbm_php_mask_inline_html(a, source, source_len);
3332+
}
3333+
return source;
3334+
}
3335+
33243336
/* Initial block for the per-file traversal scratch arena, chosen by measuring
33253337
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
33263338
* both this size and at 1 MB, and on most files at 256 KB, where the two
@@ -3741,6 +3753,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
37413753
return result;
37423754
}
37433755

3756+
source = extract_parse_bytes(a, language, source, source_len);
3757+
37443758
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
37453759
// merge overflows a small stack during the parse (see
37463760
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is

‎internal/cbm/cbm.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1385,6 +1385,10 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx);
13851385
// per ref()/source() call. No-op unless the file parses as SQL and actually
13861386
// contains a dbt builtin call. Defined in extract_dbt.c.
13871387
void cbm_extract_dbt(CBMExtractCtx *ctx);
1388+
// PHP source with its inline-HTML regions blanked so the php_only grammar can
1389+
// parse past `?>` (#2000). Same length and line breaks as the input; returns
1390+
// `source` itself when nothing needs rewriting. Defined in php_inline_html.c.
1391+
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len);
13881392
void cbm_extract_imports(CBMExtractCtx *ctx);
13891393
void cbm_extract_usages(CBMExtractCtx *ctx);
13901394
void cbm_extract_semantic(CBMExtractCtx *ctx);

0 commit comments

Comments
 (0)