Skip to content

Commit dd9558e

Browse files
committed
Merge origin/main into fix/manifest-walk-nul-file
Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
2 parents 79a5734 + 008b6c3 commit dd9558e

61 files changed

Lines changed: 6141 additions & 302 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ jobs:
317317
persist-credentials: false
318318

319319
- name: Download all native candidate pairs
320-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
320+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
321321
with:
322322
pattern: release-candidates-*
323323
path: ${{ runner.temp }}/release-candidate-artifacts

‎.github/workflows/_smoke.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ jobs:
103103
go-version-file: pkg/go/go.mod
104104
cache: false
105105

106-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
106+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
107107
with:
108108
name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
109109

@@ -203,7 +203,7 @@ jobs:
203203
zip
204204
coreutils
205205
206-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
206+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
207207
with:
208208
name: binaries-windows-${{ matrix.arch }}
209209

@@ -303,7 +303,7 @@ jobs:
303303
go-version-file: pkg/go/go.mod
304304
cache: false
305305

306-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
306+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
307307
with:
308308
name: binaries-linux-${{ matrix.arch }}-portable
309309

‎.github/workflows/_soak.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
7474
- name: Download selected release container
7575
if: ${{ inputs.use_release_artifacts }}
76-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
76+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
7777
with:
7878
name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
7979
path: ${{ runner.temp }}/cbm-selected-container
@@ -147,7 +147,7 @@ jobs:
147147
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
148148
- name: Download selected release container
149149
if: ${{ inputs.use_release_artifacts }}
150-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
150+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
151151
with:
152152
name: binaries-windows-amd64
153153
path: ${{ runner.temp }}/cbm-selected-container
@@ -225,7 +225,7 @@ jobs:
225225
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
226226
- name: Download selected release container
227227
if: ${{ inputs.use_release_artifacts }}
228-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
228+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
229229
with:
230230
name: binaries-windows-arm64
231231
path: ${{ runner.temp }}/cbm-selected-container
@@ -287,7 +287,7 @@ jobs:
287287
timeout-minutes: 10
288288
run: APT_RETRIES=3; [ -f /etc/apt/apt-mirrors.txt ] && APT_RETRIES=0; sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 --error-on=any update && sudo apt-get -o Acquire::Retries=$APT_RETRIES -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y python3 git
289289
- name: Download selected portable release container
290-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
290+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
291291
with:
292292
name: binaries-linux-${{ matrix.arch }}-portable
293293
path: ${{ runner.temp }}/cbm-selected-container

‎.github/workflows/_test.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ jobs:
231231
# producer, so arm64, macOS and Windows code under #ifdef has static
232232
# evidence only and those legs run every test.
233233
if: ${{ inputs.test_selection != '' && runner.os == 'Linux' && runner.arch == 'X64' }}
234-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
234+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
235235
with:
236236
name: ${{ inputs.test_selection }}
237237
path: ${{ runner.temp }}/test-selection
@@ -720,7 +720,7 @@ jobs:
720720
with:
721721
path: repo
722722

723-
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
723+
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
724724
with:
725725
pattern: shard-manifest-*
726726
path: manifests

‎.github/workflows/release.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ jobs:
185185
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
186186

187187
- name: Download only selected release containers
188-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
188+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
189189
with:
190190
pattern: binaries-*
191191
merge-multiple: true
@@ -194,7 +194,7 @@ jobs:
194194
# download above. Preserve only the three reviewable manifests that bind
195195
# candidate bytes, VT results, and the tuple-local decision.
196196
- name: Download candidate selection evidence separately
197-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
197+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
198198
with:
199199
name: release-selection-evidence
200200
path: selection-evidence
@@ -617,7 +617,7 @@ jobs:
617617
# the release-draft job as a same-run artifact (not a draft-release
618618
# download — that would need contents beyond read plus gh).
619619
- name: Fetch release checksums
620-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
620+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
621621
with:
622622
name: release-checksums
623623

‎Makefile.cbm‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,7 @@ EXTRACTION_SRCS = \
417417
$(CBM_DIR)/openapi.c \
418418
$(CBM_DIR)/extract_nextjs.c \
419419
$(CBM_DIR)/sql_values.c \
420+
$(CBM_DIR)/php_inline_html.c \
420421
$(CBM_DIR)/helpers.c \
421422
$(CBM_DIR)/cpp_branch_views.c \
422423
$(CBM_DIR)/result_compact.c \

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1024,10 +1024,12 @@ codebase-memory-mcp config set watcher_enabled false # stop the watcher thre
10241024
codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false)
10251025
codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit
10261026
codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit
1027+
codebase-memory-mcp config set index_max_rss_mb 8192 # optional worker-tree charged-memory limit
1028+
codebase-memory-mcp config set index_max_duration_seconds 3600 # optional per-request duration
10271029
codebase-memory-mcp config reset auto_index # reset to default
10281030
```
10291031

1030-
The two `index_max_*` settings default to `off`. Exceeding one fails the complete
1032+
The four `index_max_*` settings default to `off`. Exceeding one fails the complete
10311033
index attempt rather than publishing a partial graph; an existing serving index
10321034
is preserved. See [Index resource limits](docs/INDEX_RESOURCE_LIMITS.md).
10331035

‎docs/CONFIGURATION.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ Current keys:
9494
| `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. |
9595
| `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. |
9696
| `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. |
97+
| `index_max_rss_mb` | `off` | Optional maximum charged memory in MiB for the complete contained index-worker process tree (`64..1048576`). On macOS this is phys_footprint, not resident size. |
98+
| `index_max_duration_seconds` | `off` | Optional maximum wall-clock duration in seconds for the whole index request, including crash/hang recovery (`1..86400`). Like `index_max_rss_mb`, it stops a worker only before it starts to publish the new index; see [INDEX_RESOURCE_LIMITS.md](INDEX_RESOURCE_LIMITS.md). |
9799

98100
For a watched project, `index_status` names the polling method in `watch.strategy`:
99101
`git` (HEAD and dirty-state polling), `tree` (a non-git root polled by the
@@ -135,11 +137,13 @@ has not run yet).
135137
> `auto_index` still runs, and `index_repository` stays available for manual
136138
> reindexing.
137139
138-
The two `index_max_*` settings are independent and disabled by default. They
140+
The four `index_max_*` settings are independent and disabled by default. They
139141
apply to explicit indexing, automatic indexing, and watcher re-indexing, but not
140142
to `cross-repo-intelligence`, which does not scan repository source files.
141-
Equality is allowed; exceeding either setting fails the complete index request
142-
and preserves any previously serving database. See
143+
Equality is allowed; exceeding any setting fails the complete index request and
144+
preserves any previously serving database. Worker RSS covers descendants and is
145+
not the same as the internal `CBM_MEM_BUDGET_MB` allocation budget. Total
146+
duration is independent of the existing 15-minute no-log-progress timeout. See
143147
[Index resource limits](INDEX_RESOURCE_LIMITS.md) for counting, validation, and
144148
error-response details.
145149

‎docs/INDEX_RESOURCE_LIMITS.md‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Index resource limits
22

33
Index resource limits are optional operator controls for repositories whose
4-
discovery breadth is not known in advance. They are disabled by default so
5-
existing large-repository workloads retain their current behavior.
4+
discovery breadth or worker runtime is not known in advance. They are disabled
5+
by default so existing large-repository workloads retain their current behavior.
66

77
## Discovery settings
88

@@ -23,6 +23,61 @@ Values use base-10 integers. MiB means 1,048,576 bytes. Empty values, zero,
2323
negative values, suffixes, trailing characters, and values outside the stated
2424
ranges are rejected without changing the stored value.
2525

26+
## Worker settings
27+
28+
| Key | Default | Accepted value | Protects |
29+
|---|---:|---:|---|
30+
| `index_max_rss_mb` | `off` | `off` or `64..1048576` | Charged memory of the complete worker process tree |
31+
| `index_max_duration_seconds` | `off` | `off` or `1..86400` | Wall-clock duration of the whole index request |
32+
33+
```bash
34+
codebase-memory-mcp config set index_max_rss_mb 8192
35+
codebase-memory-mcp config set index_max_duration_seconds 3600
36+
```
37+
38+
RSS is the charged memory of the contained worker and every descendant, not
39+
the worker's allocation budget and not peak memory. On macOS that quantity is
40+
`phys_footprint` (the same number `cbm_mem_charged()` enforces), not
41+
`resident_size`, which still counts pages the allocator has already handed
42+
back. Linux and Windows use RSS / working set. This hard watchdog is
43+
separate from the internal `CBM_MEM_BUDGET_MB` soft budget. The supervisor
44+
samples the tree at most once every 250 milliseconds so the watchdog does not
45+
turn full process-table enumeration into a busy loop.
46+
47+
Duration is per request: the clock starts at the first worker spawn and is
48+
not reset when crash/hang recovery starts a later attempt. Continuous log
49+
progress does not reset it. It is independent of the existing 15-minute quiet
50+
timeout, which still identifies a worker that stops making progress. A
51+
duration limit shorter than that quiet timeout kills a hung worker before hang
52+
quarantine can name the in-flight file, so the next attempt may hang on the
53+
same file.
54+
55+
Equality is allowed. The first RSS or elapsed-duration observation above its
56+
limit starts the existing graceful-to-force process-tree shutdown. CBM reports
57+
terminal only after the tree is quiescent or a bounded containment failure is
58+
explicitly surfaced. Resource termination is not retried and does not
59+
quarantine a source file.
60+
61+
A worker limit never stops a worker after it has started to publish. The
62+
supervisor enforces the limits, but only the worker replaces the serving
63+
database, so the two coordinate through a per-attempt publication claim that
64+
exactly one of them can win. The worker claims it immediately before it
65+
replaces the serving database; the supervisor claims it before it stops the
66+
worker for a limit. If the limit is reached first, the worker can no longer
67+
publish and the previous index stays untouched. If the worker claimed first,
68+
the limits no longer apply to that attempt: the new index is published and the
69+
rest of the attempt, including the persistence artifact export, runs to
70+
completion. The supervisor then logs `index.supervisor.resource_limit_after_publication`
71+
instead of stopping it. With both worker limits `off` there is no claim.
72+
73+
If RSS is enabled and three consecutive probes cannot obtain any trustworthy
74+
tree measurement while the root worker is still running, CBM fails closed with
75+
`code=resource_probe_failed`. A probe is untrustworthy as soon as one member of
76+
the tree cannot be measured: only a process that exits between the listing and
77+
its measurement is skipped, because a total without a live member would
78+
under-report. On Linux the probe reads every `/proc/<pid>/stat` to find the
79+
tree's members, so a `/proc` mounted with `hidepid=1` makes it fail closed.
80+
2681
## Counting and failure semantics
2782

2883
`index_max_files` counts a file only after it passes directory pruning, ignore
@@ -55,6 +110,15 @@ The previous database remains available because publication occurs only after a
55110
complete discovery and successful staged build. If no previous database exists,
56111
`serving_index_preserved` is false.
57112

113+
Worker limit failures use the same shape with `stage=worker`,
114+
`resource=rss_bytes` and `unit=bytes`, or `resource=duration_ms` and
115+
`unit=milliseconds`. RSS measurement failures use `code=resource_probe_failed`
116+
and omit `observed`, `limit`, and `unit` because no trustworthy observation was
117+
available. For a worker failure `serving_index_preserved` describes the whole
118+
request: it is true only when a servable index existed when the request began,
119+
no attempt of the request (including an earlier one that crashed and was
120+
recovered) claimed publication, and that index still serves.
121+
58122
## Trust and compatibility
59123

60124
Limits are read from the CLI-managed `_config.db`; they are not MCP request
@@ -64,5 +128,6 @@ parent policy.
64128

65129
These settings do not replace or increase `auto_index_limit`, change the 512 MiB
66130
single-file cap, alter workspace-root authorization, or affect
67-
`cross-repo-intelligence`. With both settings `off`, discovery follows the
68-
existing unbounded path.
131+
`cross-repo-intelligence`. With all settings `off`, discovery follows the
132+
existing path and the supervisor performs no periodic RSS probe or total-duration
133+
termination.

‎internal/cbm/cbm.c‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3319,6 +3319,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
33193319
return r;
33203320
}
33213321

3322+
/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
3323+
* is parsed with its markup blanked (#2000); the rewrite keeps the length and
3324+
* every line break, so offsets and line numbers still refer to the file on
3325+
* disk. Every other language parses the file as it is. */
3326+
static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
3327+
int source_len) {
3328+
if (language == CBM_LANG_PHP) {
3329+
return cbm_php_mask_inline_html(a, source, source_len);
3330+
}
3331+
return source;
3332+
}
3333+
33223334
/* Initial block for the per-file traversal scratch arena, chosen by measuring
33233335
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
33243336
* both this size and at 1 MB, and on most files at 256 KB, where the two
@@ -3739,6 +3751,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
37393751
return result;
37403752
}
37413753

3754+
source = extract_parse_bytes(a, language, source, source_len);
3755+
37423756
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
37433757
// merge overflows a small stack during the parse (see
37443758
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is

0 commit comments

Comments
 (0)