From 8018561cfe04f81ca9181b7b33f21fec6f19df6b Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 17:35:05 +0200 Subject: [PATCH 01/13] fix(release): externalize runtime assets and harden VT verification Signed-off-by: Martin Vogel --- .github/workflows/_smoke.yml | 4 +- .github/workflows/_test.yml | 46 +- .github/workflows/dry-run.yml | 32 +- .github/workflows/release.yml | 66 +- .github/workflows/smoke.yml | 17 +- Makefile.cbm | 57 +- README.md | 31 +- SECURITY.md | 22 +- THIRD_PARTY.md | 20 +- docs/CONFIGURATION.md | 2 +- docs/index.html | 38 +- docs/llms.txt | 4 +- install.ps1 | 162 +- install.sh | 139 +- internal/cbm/cbm.c | 7 + pkg/aur/.SRCINFO | 2 +- pkg/aur/PKGBUILD | 4 +- pkg/chocolatey/codebase-memory-mcp.nuspec | 9 +- pkg/chocolatey/tools/chocolateyInstall.ps1 | 7 - pkg/glama/Dockerfile | 3 +- pkg/go/cmd/codebase-memory-mcp/main.go | 2299 +++++++++++++---- pkg/go/cmd/codebase-memory-mcp/main_test.go | 2180 +++++++++++++++- .../codebase-memory-mcp/pair_lock_other.go | 10 - .../codebase-memory-mcp/pair_lock_windows.go | 21 - .../codebase-memory-mcp/runtime_lock_other.go | 34 + .../codebase-memory-mcp/runtime_lock_unix.go | 40 + .../runtime_lock_windows.go | 197 ++ pkg/homebrew/Formula/codebase-memory-mcp.rb | 2 + pkg/npm/README.md | 16 +- pkg/npm/bin.js | 154 +- pkg/npm/install.js | 832 ++++-- pkg/npm/package.json | 2 +- pkg/npm/test/package-publication.test.js | 442 +++- pkg/npm/test/windows-launcher.test.js | 151 +- pkg/pypi/README.md | 6 +- pkg/pypi/pyproject.toml | 2 +- pkg/pypi/src/codebase_memory_mcp/_cli.py | 1149 +++++++- pkg/pypi/tests/test_cli.py | 818 +++++- pkg/scoop/codebase-memory-mcp.json | 5 +- ...usData.CodebaseMemoryMcp.locale.en-US.yaml | 12 +- scripts/README.md | 13 +- scripts/build.sh | 22 +- scripts/ci/append-vt-notes.sh | 276 +- scripts/ci/check-binary-composition.sh | 125 +- scripts/ci/check-virustotal.sh | 832 +++++- scripts/ci/extract-release-archives.sh | 903 ++++++- scripts/ci/publish-vt-evidence.sh | 63 + scripts/ci/smoke-artifact.sh | 6 +- scripts/ci/test-package-wrappers.sh | 73 + scripts/clean.sh | 3 - scripts/embed-frontend.sh | 194 -- scripts/gen-third-party-notices.sh | 5 +- scripts/gen-ui-licenses.py | 8 +- scripts/license-gate.sh | 6 +- scripts/license-policy.json | 2 +- scripts/pack-ui-assets.mjs | 211 ++ scripts/package-release.sh | 176 +- scripts/security-ui.sh | 4 +- scripts/security-vendored.sh | 34 +- scripts/setup-windows.ps1 | 4 +- scripts/setup.sh | 19 +- scripts/smoke-local.sh | 68 +- scripts/smoke-test.sh | 182 +- scripts/test-windows.ps1 | 49 +- scripts/test.sh | 38 +- scripts/vendored-checksums.txt | 989 +++++++ src/cli/cli.c | 764 ++++-- src/cli/cli.h | 5 + src/cli/client_adapter.h | 8 +- src/cli/integration_assets.c | 287 +- src/cli/integration_assets.h | 65 +- src/daemon/application.c | 76 + src/daemon/application.h | 13 + src/daemon/bootstrap.c | 1 + src/daemon/host.c | 98 +- src/daemon/ipc.c | 16 +- src/daemon/runtime.c | 15 +- src/foundation/compat_fs.c | 33 +- src/foundation/secure_random.c | 79 + src/foundation/secure_random.h | 15 + src/foundation/sha256.c | 49 + src/foundation/sha256.h | 6 + src/foundation/win_utf8.h | 171 +- src/main.c | 519 +++- src/pipeline/pass_calls.c | 5 +- src/ui/asset_manifest_stub.c | 6 + src/ui/asset_pack.c | 892 +++++++ src/ui/asset_pack.h | 111 + src/ui/asset_pack_stub.c | 98 + src/ui/config.c | 7 +- src/ui/embedded_assets.h | 25 - src/ui/embedded_stub.c | 22 - src/ui/http_server.c | 141 +- src/ui/http_server.h | 9 +- src/ui/httpd.h | 2 +- test-infrastructure/vm/vm-smoke.sh | 63 +- test-infrastructure/vm/win.sh | 32 +- tests/fixtures/ui-readiness/index.html | 10 + tests/test_c_lsp.c | 5 +- tests/test_cli.c | 954 ++++++- tests/test_daemon_application.c | 141 + tests/test_daemon_bootstrap.c | 2 + tests/test_daemon_open_readiness.py | 402 +++ tests/test_daemon_runtime.c | 109 +- tests/test_httpd.c | 80 +- tests/test_main.c | 2 +- tests/test_no_embedded_scripts_contract.sh | 93 +- tests/test_package_runtime_assets_contract.sh | 506 ++++ tests/test_pipeline.c | 5 +- ...test_release_archive_extractor_contract.sh | 505 ++++ tests/test_security.c | 3 +- tests/test_smoke_fixture_contract.sh | 354 ++- tests/test_ui.c | 567 +++- tests/test_ui_asset_pack_contract.sh | 133 + tests/test_vendored_integrity_contract.sh | 42 + tests/test_venue_parity_contract.sh | 2 +- tests/test_vm_worktree_manifest.sh | 27 +- tests/test_vt_gate_zero_tolerance_contract.sh | 297 ++- tests/test_vt_release_notes_contract.sh | 169 ++ tests/test_windows_bundle_contract.sh | 170 +- tests/windows/test_non_ascii_path.py | 219 +- tests/windows/test_windows_update_handoff.py | 17 +- 122 files changed, 19299 insertions(+), 2497 deletions(-) delete mode 100644 pkg/go/cmd/codebase-memory-mcp/pair_lock_other.go delete mode 100644 pkg/go/cmd/codebase-memory-mcp/pair_lock_windows.go create mode 100644 pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go create mode 100644 pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go create mode 100644 pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go create mode 100644 scripts/ci/publish-vt-evidence.sh create mode 100644 scripts/ci/test-package-wrappers.sh delete mode 100755 scripts/embed-frontend.sh create mode 100644 scripts/pack-ui-assets.mjs create mode 100644 src/foundation/secure_random.c create mode 100644 src/foundation/secure_random.h create mode 100644 src/ui/asset_manifest_stub.c create mode 100644 src/ui/asset_pack.c create mode 100644 src/ui/asset_pack.h create mode 100644 src/ui/asset_pack_stub.c delete mode 100644 src/ui/embedded_assets.h delete mode 100644 src/ui/embedded_stub.c create mode 100644 tests/fixtures/ui-readiness/index.html create mode 100644 tests/test_daemon_open_readiness.py create mode 100644 tests/test_package_runtime_assets_contract.sh create mode 100644 tests/test_release_archive_extractor_contract.sh create mode 100644 tests/test_ui_asset_pack_contract.sh create mode 100644 tests/test_vendored_integrity_contract.sh create mode 100644 tests/test_vt_release_notes_contract.sh diff --git a/.github/workflows/_smoke.yml b/.github/workflows/_smoke.yml index 8a466ab57a..d979697044 100644 --- a/.github/workflows/_smoke.yml +++ b/.github/workflows/_smoke.yml @@ -206,8 +206,8 @@ jobs: mkdir -p "$ARTIFACT_DIR" unzip -o "codebase-memory-mcp${SUFFIX}-windows-${ARCH}.zip" -d "$ARTIFACT_DIR" test -s "$ARTIFACT_DIR/codebase-memory-mcp.exe" - # ONE binary per platform: a payload sibling would mean the - # AV-flagged launcher stub came back. + # ONE executable per runtime set: a payload sibling would mean the + # removed launcher split came back. test ! -e "$ARTIFACT_DIR/codebase-memory-mcp.payload.exe" # Pre-flight the image from a profile-rooted directory: the shared # runner workspace is deliberately not a valid install root. diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 1223e194a5..c6d98127b6 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -174,6 +174,37 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache + # The package wrappers own their own runtime-set publication and lease-lock + # implementations. Run the same suites against both host families so Unix + # inode/flock behavior and Windows handle/link behavior are each gating, + # without adding Node/Go provisioning to every sanitizer/container leg. + test-package-wrappers: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: pkg/go/go.mod + cache: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Test Go, npm, and PyPI runtime-set wrappers + shell: bash + run: scripts/ci/test-package-wrappers.sh + # ThreadSanitizer data-race gate. The threaded production surfaces (watcher, # subprocess, httpd, pipeline, mcp, worker pool, diagnostics, the runnable # daemon-coordination paths) plus allocator concurrency all run under TSan @@ -502,7 +533,7 @@ jobs: run: repo/scripts/ci/verify-shard-union.sh manifests # Windows product-surface regression guards. Distinct from test-windows above - # (the sanitizer C suite): these drive a real product binary + embedded HTTP UI + # (the sanitizer C suite): these drive a real product binary + external UI pack # over stdio / CLI / HTTP and fail if a Windows bug already fixed on main comes # back -- non-ASCII repo paths dropping definitions (#636/#357, fixed by #700), # the PreToolUse hook augmenter no-op on drive-letter cwd (#618, fixed by #619), @@ -530,10 +561,6 @@ jobs: with: node-version: "22" - - name: Test npm wrapper failure handling - working-directory: pkg/npm - run: npm test - - name: Compiler cache (content-verified) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -543,11 +570,12 @@ jobs: ccache-guards-${{ github.ref }}- ccache-guards- - - name: Build product binary with embedded UI + - name: Build product binary with external UI assets and guard seams shell: msys2 {0} - # --with-ui builds the frontend (npm) and embeds it, so the drive-picker - # guard's HTTP UI is available. Functional gate only (no sanitizers). - run: scripts/build.sh --with-ui CC=clang CXX=clang++ + # --with-ui builds the frontend and its content-addressed asset pack, so + # the drive-picker guard's HTTP UI is available. Functional gate only + # (no sanitizers). + run: scripts/build.sh --with-ui CC=clang CXX=clang++ TEST_SEAMS=1 env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 1500M diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index bb0c51c71b..19d6f45152 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -149,22 +149,40 @@ jobs: # Same extraction the release verify job performs, so the bytes handed to # VirusTotal here are the bytes a user would download and run. - name: Extract archive artifacts - run: scripts/ci/extract-release-archives.sh assets binaries + run: | + scripts/ci/extract-release-archives.sh assets binaries \ + --expect-archives=16 \ + --expect-binaries=16 \ + --expect-packs=8 \ + --expect-runtime-files=64 - name: VirusTotal scan uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0 id: virustotal with: vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} - files: binaries/* + files: binaries/objects/* + request_rate: 4 - # Same ZERO-tolerance gate as the release. A detection here is a real - # finding even when it is a false positive: it is what the release gate - # will do, so the dry run should say so now. Resolve it upstream (vendor - # false-positive submission for the exact hashes, then re-run) — never by - # loosening this gate. + # Same ZERO-tolerance gate as the release. Every artifact must arrive + # naturally clean; any malicious or suspicious verdict fails this run. - name: Wait for VirusTotal results env: VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }} + VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv + VT_ASSOCIATIONS: binaries/associations.tsv + VT_RESULTS_PATH: binaries/vt-results.tsv + MIN_ENGINES: 50 run: scripts/ci/check-virustotal.sh + + - name: Preserve VirusTotal scope and result evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: virustotal-evidence-dry-run-${{ github.run_id }} + path: | + binaries/associations.tsv + binaries/scan-set.tsv + binaries/vt-results.tsv + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 207c61cc66..b0c9ede3b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -214,29 +214,21 @@ jobs: run: | mkdir -p assets binaries gh release download "$VERSION" --dir assets --repo "$GITHUB_REPOSITORY" --pattern '*.tar.gz' --pattern '*.zip' - for f in assets/*.tar.gz; do - NAME=$(basename "$f" .tar.gz) - tar -xzf "$f" -C binaries/ 2>/dev/null || true - [ -f binaries/codebase-memory-mcp ] && mv binaries/codebase-memory-mcp "binaries/${NAME}" - done - for f in assets/*.zip; do - NAME=$(basename "$f" .zip) - unzip -o "$f" -d binaries/ 2>/dev/null || true - [ -f binaries/codebase-memory-mcp.exe ] && mv binaries/codebase-memory-mcp.exe "binaries/${NAME}.exe" - done - cp install.sh binaries/ 2>/dev/null || true - cp install.ps1 binaries/ 2>/dev/null || true - ls -la binaries/ + scripts/ci/extract-release-archives.sh assets binaries \ + --expect-archives=16 \ + --expect-binaries=16 \ + --expect-packs=8 \ + --expect-runtime-files=64 - - name: Security audits on all release files + - name: Security audits on all unique release objects run: | - # Audit every file that ships in the release archives — binaries, - # install scripts, LICENSE, and any future companion files. + # Audit every distinct archive/member/UI-asset byte sequence. The + # associations manifest preserves every shipped archive location. # security-strings.sh detects file type and applies binary-only # rules (URL allowlist, dangerous-command detection) only to real # binaries; for shell scripts it still runs credential and base64 # pattern audits. - for f in binaries/*; do + for f in binaries/objects/*; do [ -f "$f" ] || continue echo "--- Auditing: $(basename "$f") ---" scripts/security-strings.sh "$f" @@ -247,24 +239,50 @@ jobs: id: virustotal with: vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} - files: binaries/* + files: binaries/objects/* + request_rate: 4 - # ZERO tolerance: any detection on any artifact blocks the release. - # False positives are resolved upstream (Microsoft false-positive - # submission for the exact hashes, then re-run this job — no rebuild), - # never by loosening this gate. + # ZERO tolerance: every artifact must arrive naturally clean. Any + # malicious or suspicious verdict blocks the release with no exception. - name: Wait for VirusTotal results env: VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }} VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }} + VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv + VT_ASSOCIATIONS: binaries/associations.tsv + VT_RESULTS_PATH: binaries/vt-results.tsv + MIN_ENGINES: 50 run: scripts/ci/check-virustotal.sh - # Renders per-binary sha256 + VT links into the notes; only reachable - # after the zero-tolerance gate passed, so "0 detections" is verified. + - name: Preserve VirusTotal scope and result evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: virustotal-evidence-release-${{ github.run_id }} + path: | + binaries/associations.tsv + binaries/scan-set.tsv + binaries/vt-results.tsv + if-no-files-found: warn + + - name: Publish durable public VirusTotal evidence + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ inputs.version }} + VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv + VT_ASSOCIATIONS: binaries/associations.tsv + VT_RESULTS_PATH: binaries/vt-results.tsv + run: bash scripts/ci/publish-vt-evidence.sh + + # Renders the exact scan scope plus archive SHA-256/engine evidence. It + # runs only after the exact-set zero-tolerance gate passed. - name: Append VirusTotal scan links to release notes env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERSION: ${{ inputs.version }} + VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv + VT_ASSOCIATIONS: binaries/associations.tsv + VT_RESULTS_PATH: binaries/vt-results.tsv run: bash scripts/ci/append-vt-notes.sh # ── 8. Publish package wrappers (npm + PyPI) ────────────────── diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 3db24d53cc..aa91f91c3d 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -1,5 +1,6 @@ -# Smoke invariants — "the shipped binary does not fail" — across the WIDEST set of -# GitHub-hosted runners. Builds the prod binary and runs scripts/smoke-invariants.sh +# Smoke invariants across the WIDEST set of GitHub-hosted runners. Builds the +# production source path with explicit fault-injection test seams and runs +# scripts/smoke-invariants.sh # (version/help, MCP initialize handshake [#513], all 14 tools invocable, malformed- # input resilience, clean EOF exit, shared-lib resolution, install dry-run). # @@ -54,8 +55,8 @@ jobs: - name: Install deps (Linux) if: startsWith(matrix.os, 'ubuntu') run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git - - name: Build (prod binary) - run: scripts/build.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} + - name: Build (production path with explicit test seams) + run: scripts/build.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} TEST_SEAMS=1 - name: Smoke invariants run: | chmod +x scripts/smoke-invariants.sh @@ -82,9 +83,9 @@ jobs: make git coreutils - - name: Build (prod binary) + - name: Build (production path with explicit test seams) shell: msys2 {0} - run: scripts/build.sh CC=clang CXX=clang++ + run: scripts/build.sh CC=clang CXX=clang++ TEST_SEAMS=1 - name: Smoke invariants shell: msys2 {0} run: | @@ -112,9 +113,9 @@ jobs: make git coreutils - - name: Build (prod binary) + - name: Build (production path with explicit test seams) shell: msys2 {0} - run: scripts/build.sh CC=clang CXX=clang++ + run: scripts/build.sh CC=clang CXX=clang++ TEST_SEAMS=1 - name: Smoke invariants shell: msys2 {0} run: | diff --git a/Makefile.cbm b/Makefile.cbm index f7eef1d781..a7c4b42b16 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -148,7 +148,7 @@ ifeq ($(IS_MINGW),yes) # two Windows builds of identical source are never byte-identical and every # release is a brand-new file to a reputation system. Same motivation as the # mimalloc __DATE__ patch; ELF and Mach-O have no equivalent field to clear. -WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static +WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -lbcrypt -Wl,--allow-multiple-definition -Wl,--stack,8388608 -Wl,--no-insert-timestamp -static endif # STATIC=1 produces a fully static binary (for Alpine/musl portable builds) @@ -205,6 +205,7 @@ FOUNDATION_SRCS = \ src/foundation/limits.c \ src/foundation/subprocess.c \ src/foundation/sha256.c \ + src/foundation/secure_random.c \ src/foundation/macos_acl.c \ src/foundation/private_file_lock.c \ src/foundation/lock_registry.c @@ -254,7 +255,7 @@ TS_RUNTIME_DEPS = \ $(wildcard $(CBM_DIR)/vendored/ts_runtime/src/portable/*.h) \ $(wildcard $(CBM_DIR)/vendored/ts_runtime/src/unicode/*.h) -# All 64 grammar shim files +# All 159 grammar shim files GRAMMAR_SRCS = $(wildcard $(CBM_DIR)/grammar_*.c) # LZ4 + Aho-Corasick @@ -359,13 +360,15 @@ CLI_SRCS = src/cli/cli.c src/cli/progress_sink.c src/cli/hook_augment.c src/cli/ src/cli/config_json_like.c src/cli/config_toml_edit.c src/cli/config_yaml_edit.c \ src/cli/config_text_edit.c src/cli/activation_transaction.c -# UI module (graph visualization) +# UI module (graph visualization). Production-standard builds deliberately link +# only the no-I/O pack stub. Test binaries substitute the real parser plus the +# empty manifest so parser/install behavior remains covered without a UI pack. UI_SRCS = \ src/ui/config.c \ src/ui/http_server.c \ src/ui/layout3d.c \ src/ui/httpd.c \ - src/ui/embedded_stub.c + src/ui/asset_pack_stub.c # mimalloc (vendored, global allocator override) # @@ -453,6 +456,7 @@ YYJSON_SRC = vendored/yyjson/yyjson.c # All production sources PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC) +TEST_PROD_SRCS = $(subst src/ui/asset_pack_stub.c,src/ui/asset_pack.c src/ui/asset_manifest_stub.c,$(PROD_SRCS)) EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \ $(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) @@ -722,7 +726,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security +.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend ui-pack clean-c lint lint-tidy lint-cppcheck lint-format security $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -865,9 +869,9 @@ $(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUIL OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ) OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ) -$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) +$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) $(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \ - $(ALL_TEST_SRCS) $(PROD_SRCS) \ + $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) \ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ $(OBJS_VENDORED_TEST) \ $(LDFLAGS_TEST) @@ -893,9 +897,9 @@ test-focused: $(BUILD_DIR)/test-runner # Mirrors test-runner's link line but uses repro_main.c (own main + counters) # and TEST_REPRO_SRCS instead of ALL_TEST_SRCS. Exits non-zero while any bug is # still reproduced (the expected state); bug-repro.yml surfaces it as a board. -$(BUILD_DIR)/test-repro-runner: $(TEST_REPRO_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) +$(BUILD_DIR)/test-repro-runner: $(TEST_REPRO_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR) $(CC) $(CFLAGS_TEST) -Itests -o $@ \ - $(TEST_REPRO_SRCS) $(PROD_SRCS) \ + $(TEST_REPRO_SRCS) $(TEST_PROD_SRCS) \ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ $(OBJS_VENDORED_TEST) \ $(LDFLAGS_TEST) @@ -953,9 +957,9 @@ TSAN_OPTIONS ?= halt_on_error=1:report_thread_leaks=0 # Normal ASan/native/soak paths remain uncapped. High-worker TSan diagnostics # invoke test-runner-tsan directly so release gates cannot drift accidentally. -$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) $(PROJECT_HDRS) | $(BUILD_DIR) +$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) $(PROJECT_HDRS) | $(BUILD_DIR) $(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \ - $(ALL_TEST_SRCS) $(PROD_SRCS) \ + $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) \ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ $(OBJS_VENDORED_TSAN) \ $(LDFLAGS_TSAN) @@ -1042,33 +1046,40 @@ $(BUILD_DIR)/codebase-memory-mcp: $(MAIN_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $( cbm: $(BUILD_DIR)/codebase-memory-mcp @echo "Built: $(BUILD_DIR)/codebase-memory-mcp" -# ── Build with embedded UI (requires Node.js) ─────────────────── +# ── Build with external UI asset pack (requires Node.js) ──────── -# Swap embedded_stub.c for the generated embedded_assets.c -UI_SRCS_WITH_ASSETS = $(subst src/ui/embedded_stub.c,src/ui/embedded_assets.c,$(UI_SRCS)) -PROD_SRCS_WITH_ASSETS = $(subst src/ui/embedded_stub.c,src/ui/embedded_assets.c,$(PROD_SRCS)) - -# Embedded asset object files (generated by embed script) -EMBED_OBJS = $(wildcard $(BUILD_DIR)/embedded/embed_*.o) +# The native image contains only a generated manifest (pack basename, SHA-256 +# and byte length). Frontend bytes stay in a content-addressed sidecar next to +# the binary so release scanners can inspect them independently. +UI_ASSET_MANIFEST = $(BUILD_DIR)/generated/ui_asset_manifest.c +UI_ASSET_DIST ?= graph-ui/dist +UI_ASSET_PREBUILT ?= 0 +ifeq ($(UI_ASSET_PREBUILT),1) +UI_ASSET_BUILD_PREREQ := +else +UI_ASSET_BUILD_PREREQ := frontend +endif +PROD_SRCS_WITH_ASSETS = $(subst src/ui/asset_pack_stub.c,src/ui/asset_pack.c $(UI_ASSET_MANIFEST),$(PROD_SRCS)) frontend: cd graph-ui && npm ci && npm run build -embed: frontend - scripts/embed-frontend.sh graph-ui/dist $(BUILD_DIR)/embedded +$(UI_ASSET_MANIFEST): $(UI_ASSET_BUILD_PREREQ) scripts/pack-ui-assets.mjs | $(BUILD_DIR) + node scripts/pack-ui-assets.mjs $(UI_ASSET_DIST) $(BUILD_DIR) $@ + +ui-pack: $(UI_ASSET_MANIFEST) # $(INTEGRATIONS_HASH_HDR): generated header included by integration_assets.c. # The standard target gets it via $(PROJECT_HDRS); the UI target compiles in one # shot and must list it too, or the build fails with "cbm_integrations_hash.h # file not found" (seen on the Windows UI build). -cbm-with-ui: embed $(INTEGRATIONS_HASH_HDR) $(OBJS_VENDORED_PROD) +cbm-with-ui: ui-pack $(INTEGRATIONS_HASH_HDR) $(OBJS_VENDORED_PROD) $(CC) $(CFLAGS_PROD) -o $(BUILD_DIR)/codebase-memory-mcp \ $(MAIN_SRC) $(PROD_SRCS_WITH_ASSETS) \ $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ $(OBJS_VENDORED_PROD) \ - $(wildcard $(BUILD_DIR)/embedded/embed_*.o) \ $(LDFLAGS) - @echo "Built with UI: $(BUILD_DIR)/codebase-memory-mcp" + @echo "Built with UI asset pack: $(BUILD_DIR)/codebase-memory-mcp" clean-c: export BUILD_DIR := $(BUILD_DIR) clean-c: diff --git a/README.md b/README.md index 926d13d7ba..203d6f1c9e 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,20 @@ [![Languages](https://img.shields.io/badge/languages-158-orange)](https://github.com/DeusData/codebase-memory-mcp) [![Hybrid LSP](https://img.shields.io/badge/Hybrid_LSP-10_languages-blue)](#hybrid-lsp) [![Agents](https://img.shields.io/badge/agent_surfaces-43-purple)](https://github.com/DeusData/codebase-memory-mcp) -[![Pure C](https://img.shields.io/badge/pure_C-zero_dependencies-blue)](https://github.com/DeusData/codebase-memory-mcp) +[![Pure C](https://img.shields.io/badge/pure_C-no_language_runtime-blue)](https://github.com/DeusData/codebase-memory-mcp) [![Platform](https://img.shields.io/badge/macOS_%7C_Linux_%7C_Windows-supported-lightgrey)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/DeusData/codebase-memory-mcp/badge)](https://scorecard.dev/viewer/?uri=github.com/DeusData/codebase-memory-mcp) [![SLSA 3](https://slsa.dev/images/gh-badge-level3.svg)](https://slsa.dev) [![VirusTotal](https://img.shields.io/badge/VirusTotal-scanned_every_release-brightgreen?logo=virustotal)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) [![arXiv](https://img.shields.io/badge/arXiv-2603.27277-b31b1b?logo=arxiv)](https://arxiv.org/abs/2603.27277) -**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a single static binary for macOS, Linux, and Windows — download, run `install`, done. +**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a native executable with a small verified runtime-asset set for macOS, Linux, and Windows — download, run `install`, done. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 158 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. Zero dependencies. Plug and play across 43 supported automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across all 158 languages, enhanced with [**Hybrid LSP** semantic type resolution](#hybrid-lsp) for Python, TypeScript / JavaScript / JSX / TSX, PHP, C#, Go, C, C++, Java, Kotlin, Rust, and Perl — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 15 MCP tools. No language runtime, hosted service, or API key. Plug and play across 43 supported automatic/conditional client surfaces. > **Research** — The design and benchmarks behind this project are described in the preprint [*Codebase-Memory: Tree-Sitter-Based Knowledge Graphs for LLM Code Exploration via MCP*](https://arxiv.org/abs/2603.27277) (arXiv:2603.27277). Evaluated across 31 real-world repositories: 83% answer quality, 10× fewer tokens, 2.1× fewer tool calls vs. file-by-file exploration. -> **Security & Trust** — This tool reads your codebase and writes to your agent configuration files. That is what it is designed to do. If you prefer to audit before running, the [full source is here](https://github.com/DeusData/codebase-memory-mcp) — every release binary is signed, checksummed, and scanned by 70+ antivirus engines. All processing happens 100% locally; your code never leaves your machine. Found a security issue? We want to know — see [SECURITY.md](SECURITY.md). Security is Priority #1 for us. +> **Security & Trust** — This tool reads your codebase and writes to your agent configuration files. That is what it is designed to do. If you prefer to audit before running, the [full source is here](https://github.com/DeusData/codebase-memory-mcp). Release archives are signed and checksummed; every distinct shipped archive, member, and UI-asset byte object is submitted to VirusTotal. The release notes record the measured engine count and exact 0/N result; publication requires at least 50 decisive engine results with zero malicious and zero suspicious verdicts. All processing happens 100% locally; your code never leaves your machine. Found a security issue? We want to know — see [SECURITY.md](SECURITY.md). Security is Priority #1 for us.

Graph visualization UI showing the codebase-memory-mcp knowledge graph @@ -31,7 +31,7 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si ## Why codebase-memory-mcp - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline: LZ4 compression, in-memory SQLite, fused Aho-Corasick pattern matching. Memory released after indexing. -- **Plug and play** — single static binary for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). No Docker, no runtime dependencies, no API keys. Download → `install` → restart agent → done. +- **Plug and play** — native executable plus authenticated release-owned assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). The native install needs no Docker, language runtime, or API keys. Download → `install` → restart agent → done. - **158 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. One graph query replaces dozens of grep/read cycles. - **43 supported automatic/conditional client surfaces** — `install` configures detected clients and safely activates conditional clients only when their documented platform, marker, or explicit existing config path is present. See [Multi-Agent Support](#multi-agent-support) for the complete matrix and manual/UI-only boundaries. @@ -120,17 +120,18 @@ All active CBM processes must run the exact same version, executable build, coor The native `install`, `update`, and `uninstall` commands are the deliberate exception to that conflict rule. Download, verification, and private same-filesystem staging happen first so a bad candidate never disrupts active work. Activation then publishes account-wide maintenance intent, asks the daemon and every temporary local operation to cancel, and waits to a finite deadline for all coordinated CBM processes to exit. It holds the admission and lifetime barriers exclusively while changing the active binary, configuration, PATH, or indexes. New CBM work cannot enter during this window. Activation progress and results are recorded in `${CBM_CACHE_DIR}/logs/activation-events.ndjson`, and a successful command tells you to restart open coding-agent sessions so they launch the activated build. -Package-manager setup (npm, PyPI, or Go) only verifies and atomically publishes that package's private cached binary; it does not replace the active native installation and therefore does not stop running CBM sessions. When that cached binary is executed, it still enters the same exact-build admission barrier. The shell and PowerShell installers invoke the verified candidate's native `install` command, so they do receive the full account-wide activation guarantee. +Package-manager setup (npm, PyPI, or Go) verifies and publishes a coherent private cached runtime set. Sidecars are replaced before the executable with per-file atomic renames; an interrupted multi-file publication is detected and repaired on the next launch rather than being described as one crash-atomic filesystem transaction. It does not replace the active native installation and therefore does not stop running CBM sessions. When that cached binary is executed, it still enters the same exact-build admission barrier. The shell and PowerShell installers invoke the verified candidate's native `install` command, so they do receive the full account-wide activation guarantee. The ordinary `cli` mode is intentionally separate: it runs one command locally and never starts or connects to the coordination daemon, registers a daemon session, or starts watchers/UI. Its only shared state is the OS admission barrier plus per-project locks for graph mutations. While the command is running, a temporary monitor lets activation cancel that operation and its supervised worker safely; the monitor exits with the command and never becomes a standing daemon. See [CLI Mode](#cli-mode) for details. ### Graph Visualization UI -The UI ships as a separate `ui` build (it embeds the frontend). The default install on every channel is the lean, headless server; opt into the UI build with: +The UI currently ships as a separate `ui` build whose executable embeds only a content-addressed manifest; the deterministic frontend pack is a verified sidecar beside it. The default install on every channel is the lean, headless server; opt into the UI build with: - **install.sh:** add `--ui` (see [Quick Start](#quick-start)) - **npm:** `CBM_VARIANT=ui npm install -g codebase-memory-mcp` - **PyPI:** `CBM_VARIANT=ui pip install codebase-memory-mcp` +- **Go:** after `go install github.com/DeusData/codebase-memory-mcp/pkg/go/cmd/codebase-memory-mcp@latest`, run the wrapper with `CBM_VARIANT=ui` - **Manual:** download the `codebase-memory-mcp-ui--` archive Then run it: @@ -231,7 +232,7 @@ The install script placed beside the binary is **reported, not deleted** — uni - **RAM-first pipeline**: LZ4 compression, in-memory SQLite, single dump at end. Memory released after. ### Distribution & operation -- **Single static binary, zero infrastructure**: SQLite-backed, persists to `~/.cache/codebase-memory-mcp/` +- **Native runtime set, zero infrastructure services**: SQLite-backed, persists to `~/.cache/codebase-memory-mcp/` - **Auto-sync**: Background watcher detects file changes and re-indexes automatically - **Route nodes**: REST endpoints are first-class graph entities - **CLI mode**: `codebase-memory-mcp cli search_graph '{"project": "my-project", "name_pattern": ".*Handler.*"}'` @@ -318,7 +319,7 @@ When you open a memory/performance issue, **attach the `.ndjson` trajectory** | Linux (ARM64) | `codebase-memory-mcp-linux-arm64.tar.gz` | `codebase-memory-mcp-ui-linux-arm64.tar.gz` | | Windows (x86_64) | `codebase-memory-mcp-windows-amd64.zip` | `codebase-memory-mcp-ui-windows-amd64.zip` | -Every release includes `checksums.txt` with SHA-256 hashes. All binaries are statically linked — no shared library dependencies. +Every release includes `checksums.txt` with SHA-256 hashes. Keep the native executable together with the authenticated `cbm-integrations.json` asset and, for the UI variant, its single content-addressed `cbm-ui-.pack`. Linux `-portable` archives contain the fully static builds; ordinary platform archives use their native system ABI. > **Windows note**: SmartScreen may show a warning for unsigned software. Click **"More info"** → **"Run anyway"**. Verify integrity with `checksums.txt`. @@ -381,7 +382,7 @@ scripts/build.sh --with-ui # with graph visualization # Binary at: build/c/codebase-memory-mcp (codebase-memory-mcp.exe on Windows) ``` -Every platform builds **one self-contained binary** — no runtime dependencies, no companion executables to keep alongside it. +Every platform builds a **verified runtime set**: the native executable, `cbm-integrations.json`, and—when `--with-ui` is selected—exactly one content-addressed `cbm-ui-.pack`. These files are authenticated and published together; do not separate the executable from its sidecars. Run the test suite (6,768 tests across 120 suites): @@ -731,7 +732,7 @@ SQLite databases stored at `~/.cache/codebase-memory-mcp/`. Persists across rest Tree-sitter alone gives a syntactic AST. That handles naming, structure, and call sites well, but it can't tell you that `user.profile.display_name()` resolves to `Profile.display_name` declared three modules away — tree-sitter doesn't track imports, generics, inheritance, or stdlib types. -codebase-memory-mcp ships a **lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers** (tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, rust-analyzer), embedded directly into the static binary. No language server process, no per-project setup, no API key. We call this layer **Hybrid LSP**: it runs alongside tree-sitter on every parse and refines invocation resolution (`CALLS` / `RESOLVED_CALLS`) and callable-value resolution (`CALL_REFERENCE`, with ambiguous values retained as `USAGE`) using type information, so the resulting graph mirrors what an IDE "Go to Definition" would resolve. +codebase-memory-mcp ships a **lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers** (tsserver / typescript-go, pyright, gopls, Roslyn, Eclipse JDT, rust-analyzer), embedded directly into the native executable. No language server process, no per-project setup, no API key. We call this layer **Hybrid LSP**: it runs alongside tree-sitter on every parse and refines invocation resolution (`CALLS` / `RESOLVED_CALLS`) and callable-value resolution (`CALL_REFERENCE`, with ambiguous values retained as `USAGE`) using type information, so the resulting graph mirrors what an IDE "Go to Definition" would resolve. **Languages with full Hybrid LSP:** @@ -781,21 +782,21 @@ src/ discover/ File discovery (.gitignore, .cbmignore, symlink handling) watcher/ Background auto-sync (git polling, adaptive intervals) traces/ Runtime trace ingestion - ui/ Embedded HTTP server + 3D graph visualization + ui/ Local HTTP server + verified external 3D-UI asset pack foundation/ Platform abstractions (threads, filesystem, logging, memory) internal/cbm/ Vendored tree-sitter grammars (158 languages) + AST extraction engine ``` ## Security -Every release binary is verified through a multi-layer pipeline before publication: +Every release is verified through a multi-layer pipeline before publication: -- **VirusTotal** — all binaries scanned by 70+ antivirus engines (zero detections required to publish) +- **VirusTotal** — every release object is scanned (minimum 50 decisive engines; zero malicious and zero suspicious required). Release notes summarize the archive rows and link durable public association, exact-scan-set, and per-object result files containing every measured engine count and VirusTotal report URL. - **SLSA Level 3** — cryptographic build provenance generated by the trusted GitHub Actions build workflow; verify with `gh attestation verify --repo DeusData/codebase-memory-mcp --signer-workflow DeusData/codebase-memory-mcp/.github/workflows/_build.yml` - **Sigstore cosign** — keyless signatures on all artifacts; bundles included in every release - **SHA-256 checksums** — `checksums.txt` published with every release; verified by both install scripts before extraction - **CodeQL SAST** — blocks release pipeline if any open alerts remain -- **Zero runtime dependencies** — no transitive supply chain; all libraries vendored at compile time +- **No language-runtime dependency chain** — libraries are vendored at compile time; the small release-owned runtime assets are checksum-verified and content-addressed ### v0.7.0 VirusTotal scans diff --git a/SECURITY.md b/SECURITY.md index d00b3c5568..37a61dcb6d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,7 +4,7 @@ codebase-memory-mcp interacts deeply with your filesystem. It reads source files across your entire codebase, writes to agent configuration files, and spawns background processes. This is inherent to what it does — not a bug. -**If you are uncomfortable with these access patterns**, please audit the source code before running. The full source is available in this repository. Release binaries produced by the current release pipeline are verifiably built from this source and can be independently verified via SLSA Build Level 3 provenance, Sigstore signatures, and SHA-256 checksums (see [Verification](#verification) below). +**If you are uncomfortable with these access patterns**, please audit the source code before running. The full source is available in this repository. Release archives produced by the current release pipeline are verifiably built from this source and can be independently verified via SLSA Build Level 3 provenance, Sigstore signatures, and SHA-256 checksums (see [Verification](#verification) below). Each archive contains a native executable and its authenticated release-owned runtime assets. We are humans and can make mistakes. We take security seriously — it is Priority #1 for this project — but we cannot guarantee perfection. By using this software you accept responsibility for evaluating whether it meets your own security requirements. @@ -95,7 +95,7 @@ disclosure. Research conducted under this policy is considered authorised. ## Security Measures -This project implements multiple layers of security verification. Every release binary must pass all checks before users can download it (draft → verify → publish flow). +This project implements multiple layers of security verification. Every release archive and its extracted runtime objects must pass all checks before users can download it (draft → verify → publish flow). ### Build-Time (CI — every commit) @@ -122,11 +122,11 @@ This project implements multiple layers of security verification. Every release Releases are created as **drafts** (invisible to users) and only published after all verification passes: -1. **SLSA Build Level 3 provenance for release binaries** — cryptographic attestation generated inside the trusted GitHub Actions build workflow immediately after each release archive is produced +1. **SLSA Build Level 3 provenance for release archives** — cryptographic attestation generated inside the trusted GitHub Actions build workflow immediately after each release archive is produced 2. **Sigstore cosign signing** — keyless digital signatures verifiable by anyone 3. **SBOM** — Software Bill of Materials (SPDX) listing all vendored dependencies 4. **SHA-256 checksums** — published with every release -5. **VirusTotal scanning** — all binaries scanned by 70+ antivirus engines (zero-tolerance: any detection blocks the release) +5. **VirusTotal scanning** — every distinct shipped archive, member, and UI-asset byte object is scanned. Each completed analysis must contain at least 50 decisive engine results, zero malicious verdicts, and zero suspicious verdicts. Release notes summarize archive results and link durable public association, exact-scan-set, per-object-result, and evidence-checksum assets. 6. **OpenSSF Scorecard** — repository security health score Scope of the SLSA claim: this is a build provenance claim for release @@ -137,7 +137,9 @@ not mean the source code is vulnerability-free or that maintainers cannot change source. Consumers should verify the signer workflow, not only repository ownership. -If ANY antivirus engine flags ANY binary, the release stays as a draft and is not published until the issue is investigated and resolved. +If any antivirus engine flags any scanned release object, the release stays as a +draft and is not published until the issue is investigated and resolved. There +is no exception path in this release gate. ### Code-Level Defenses @@ -146,14 +148,16 @@ If ANY antivirus engine flags ANY binary, the release stays as a draft and is no - **CORS locked to localhost** — graph UI only accessible from localhost origins - **Path containment** — `realpath()` check prevents reading files outside project root - **Process-kill restriction** — only server-spawned PIDs can be terminated -- **SHA-256 checksum verification** — update command verifies downloaded binary before installing +- **Release-set verification** — installers verify downloaded archives, then the + candidate executable authenticates its adjacent runtime assets before + activation ### Verification -Users can independently verify any release binary: +Users can independently verify any release archive and the runtime set it contains: ```bash -# SLSA Build Level 3 provenance for release binaries +# SLSA Build Level 3 provenance for release archives gh attestation verify \ --repo DeusData/codebase-memory-mcp \ --signer-workflow DeusData/codebase-memory-mcp/.github/workflows/_build.yml @@ -164,7 +168,7 @@ cosign verify-blob --bundle .bundle # SHA-256 checksum sha256sum -c checksums.txt -# VirusTotal (upload binary or check the report links in the release notes) +# VirusTotal (follow the durable per-object results link in the release notes) # https://www.virustotal.com/ ``` diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index e0a1fc82e0..f33d0ce834 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -66,7 +66,7 @@ License summary: - **Copyright:** (c) 2025 InterSystems Corporation - **Vendored at:** `internal/cbm/vendored/grammars/objectscript_udl/`, `internal/cbm/vendored/grammars/objectscript_routine/` - **Pinned commit:** `a7ffcdf` -- **Notes:** InterSystems-maintained grammar for the ObjectScript language (InterSystems IRIS / Caché). Vendor-maintained; not in nvim-treesitter or Helix registries. Each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` (verbatim copy of upstream `common/scanner.h`). +- **Notes:** InterSystems-maintained grammar for the ObjectScript language (InterSystems IRIS / Caché). Vendor-maintained; not in nvim-treesitter or Helix registries. Each `scanner.c`'s upstream `#include "../../common/scanner.h"` is repointed to a per-directory `objectscript_common.h` copied from upstream `common/scanner.h`; two loop counters in that copy are widened from `uint8_t` to `int` as documented in `internal/cbm/vendored/grammars/MANIFEST.md`. ## Vendored C/C++ Libraries @@ -142,13 +142,17 @@ follows: documentation and language specifications; no upstream source code was extracted or transcribed. -## Embedded Graph UI +## External Graph UI asset pack -Release binaries built with `--with-ui` embed the compiled `graph-ui/` -frontend bundle. Its npm dependencies (React, three.js, @react-three/*, -radix-ui, lucide-react, tailwindcss, and friends) are all under permissive -licenses (MIT / ISC / Apache-2.0 / Zlib); the exact set is recorded in +Release builds made with `--with-ui` ship the compiled `graph-ui/` frontend in +a deterministic, content-addressed `cbm-ui-.pack` beside the native +executable. The executable contains only the expected pack name, size, and +SHA-256; it does not contain the HTML, JavaScript, or CSS payload. + +The pack's npm dependencies (React, three.js, @react-three/*, radix-ui, +lucide-react, tailwindcss, and friends) are all under permissive licenses +(MIT / ISC / Apache-2.0 / Zlib). The exact set is recorded in `graph-ui/package.json` and `graph-ui/package-lock.json`, and the per-package license texts of the production bundle are appended to the -`THIRD_PARTY_NOTICES.md` shipped inside the `-ui` release archives -(generated by `scripts/gen-ui-licenses.py`). +`THIRD_PARTY_NOTICES.md` shipped inside each `-ui` release archive (generated +by `scripts/gen-ui-licenses.py`). diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index e0a589536a..fae2af19ee 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -106,7 +106,7 @@ Current format: Notes: -- If the UI-enabled binary has embedded assets and no UI config file exists yet, the UI auto-enables on first run. +- If a UI-enabled binary finds its verified external asset pack and no UI config file exists yet, the UI auto-enables on first run. Missing or invalid assets leave the MCP/daemon service available and keep the UI disabled. - `CBM_CACHE_DIR` changes both the UI config location and the runtime settings database location. - CBM resolves `CBM_CACHE_DIR` to one canonical per-account cache root. A process configured with a different root fails while any CBM session or command is active; close them before switching roots. diff --git a/docs/index.html b/docs/index.html index 28a09a9d79..a8d7d35cc8 100644 --- a/docs/index.html +++ b/docs/index.html @@ -4,7 +4,7 @@ codebase-memory-mcp — Code Intelligence Knowledge Graph for AI Coding Agents - + @@ -12,7 +12,7 @@ - + @@ -24,7 +24,7 @@ - + @@ -41,7 +41,7 @@ "applicationSubCategory": "Model Context Protocol (MCP) server", "operatingSystem": "macOS, Linux, Windows", "softwareVersion": "0.8.1", - "description": "An open-source MCP server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links, so AI coding agents answer structural questions with roughly 120x fewer tokens than file-by-file search. Parses 158 languages via tree-sitter with Hybrid LSP semantic type resolution for 9 language families. Ships as a single static C binary with zero runtime dependencies.", + "description": "An open-source MCP server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links, so AI coding agents answer structural questions with roughly 120x fewer tokens than file-by-file search. Parses 158 languages via tree-sitter with Hybrid LSP semantic type resolution for 9 language families. Ships as a native executable with authenticated release-owned assets; the native install needs no language runtime.", "url": "https://deusdata.github.io/codebase-memory-mcp/", "downloadUrl": "https://github.com/DeusData/codebase-memory-mcp/releases/latest", "codeRepository": "https://github.com/DeusData/codebase-memory-mcp", @@ -125,7 +125,7 @@ "name": "What is codebase-memory-mcp?", "acceptedAnswer": { "@type": "Answer", - "text": "codebase-memory-mcp is an open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query that graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens. It parses 158 languages and ships as a single static C binary with no runtime dependencies." + "text": "codebase-memory-mcp is an open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query that graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens. It parses 158 languages and ships as a native executable with authenticated release-owned assets; the native install needs no language runtime." } }, { @@ -165,7 +165,7 @@ "name": "Does codebase-memory-mcp send my code anywhere?", "acceptedAnswer": { "@type": "Answer", - "text": "No. All indexing and querying happen 100% locally on your machine. There is no embedded LLM and no API key — your MCP client acts as the intelligence layer. Release binaries are signed, checksummed, and scanned by 70+ antivirus engines." + "text": "No. All indexing and querying happen 100% locally on your machine. There is no embedded LLM and no API key — your MCP client acts as the intelligence layer. Release archives are signed and checksummed, and every distinct shipped archive, member, and UI-asset byte object is submitted to VirusTotal; release notes record its measured 0/N result, with at least 50 decisive engines and zero malicious or suspicious verdicts required." } }, { @@ -444,7 +444,7 @@

codebase-memory-mcp

The fastest, most efficient code intelligence engine for AI coding agents. It indexes any repository into a persistent knowledge graph — full-indexing an average repo in seconds and the Linux kernel in 3 minutes — so your agent answers structural questions with ~120x fewer tokens. - Tree-sitter parsing across 158 languages, Hybrid LSP type resolution, single static C binary. + Tree-sitter parsing across 158 languages, Hybrid LSP type resolution, native executable with verified assets.

~120x
fewer tokens
@@ -483,7 +483,8 @@

What is codebase-memory-mcp?

Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. Instead of reading files one at a time, an AI coding agent queries the graph — answering structural questions with roughly 120x fewer - tokens. It parses 158 languages and ships as a single static C binary with zero runtime dependencies. + tokens. It parses 158 languages and ships as a native executable with authenticated release-owned assets; + the native install needs no language runtime.

It is a structural-analysis backend, not a chatbot: there is no embedded LLM and no API key. Your @@ -515,8 +516,8 @@

Why do AI agents waste tokens exploring code?

How do I install codebase-memory-mcp?

- Install with a single command, then tell your agent to index the project. It is a single static C binary for - macOS, Linux, and Windows — no Docker, no runtime dependencies, no API key. + Install with a single command, then tell your agent to index the project. The native runtime set contains + an executable with verified assets for macOS, Linux, and Windows — no Docker, language runtime, or API key.

# 1. One-line install (macOS / Linux). Add --ui for the 3D graph UI.
@@ -584,7 +585,7 @@

What is Hybrid LSP?

codebase-memory-mcp ships a lightweight C implementation of language type-resolution algorithms, structurally inspired by and compatible with major language servers — tsserver/typescript-go, pyright, gopls, Roslyn, Eclipse JDT, and rust-analyzer — - embedded directly into the single static C binary. There is no language-server process, no + embedded directly into the native executable. There is no language-server process, no per-project setup, and no API key. This layer runs alongside tree-sitter on every parse and refines CALLS, USAGE, and RESOLVED_CALLS edges with type information, so the graph mirrors what an IDE "Go to Definition" would resolve. @@ -706,8 +707,8 @@

Hybrid LSP type resolution

Language-server-grade type inference for Python, TS/JS, PHP, C#, Go, C/C++, Java, Kotlin, and Rust — embedded in the binary, no server process or per-project setup.

-

Pure C, zero dependencies

-

A single static C binary for macOS, Linux, and Windows. No Docker, no runtime, no API keys. Download, run install, done.

+

Pure C, no language runtime

+

A native executable with authenticated release-owned assets for macOS, Linux, and Windows. The native install needs no Docker, language runtime, or API keys; fully static Linux builds are available in the -portable archives.

Call-graph tracing

@@ -735,7 +736,7 @@

Team-shared graph artifact

3D graph visualization

-

An optional UI binary serves an interactive 3D graph at localhost:9749 to explore nodes, edges, and clusters visually.

+

The optional UI variant adds one verified, content-addressed frontend pack and serves an interactive 3D graph at localhost:9749 to explore nodes, edges, and clusters visually.

15 MCP tools

@@ -790,7 +791,9 @@

Frequently asked questions

Does codebase-memory-mcp send my code anywhere?

No. All indexing and querying happen 100% locally. There is no embedded LLM and no API key. Release - binaries are signed, checksummed, and scanned by 70+ antivirus engines.

+ archives are signed and checksummed, and every distinct shipped archive, member, and UI-asset byte object + is submitted to VirusTotal. Release notes + record its measured 0/N result; at least 50 decisive engines and zero malicious or suspicious verdicts are required.

Does it support semantic or natural-language code search?

Yes. Alongside structural and full-text search, search_graph's semantic_query @@ -800,8 +803,9 @@

Does it support semantic or natural-language code search?

edges between similar functions and SIMILAR_TO edges for near-clone detection.

Do I need Docker or a runtime?

-

No. It is a single static C binary with zero runtime dependencies for macOS (arm64/amd64), Linux - (arm64/amd64), and Windows (amd64).

+

The native install needs no language runtime. It is a native executable with authenticated release-owned + assets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64); npm, PyPI, and Go wrapper + installations naturally use their package runtime.

How does it stay up to date as I edit code?

A background watcher detects file changes and re-indexes incrementally — typically a sub-millisecond diff --git a/docs/llms.txt b/docs/llms.txt index 32eb41640d..f7edd0316f 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -1,6 +1,6 @@ # codebase-memory-mcp -> An open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query the graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens (~3,400 vs ~412,000 across five structural queries). Parses 158 languages via vendored tree-sitter grammars, with Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust. It also generates semantic graph edges (SEMANTICALLY_RELATED for vocabulary-mismatch matches, SIMILAR_TO for near-clone detection) and supports semantic vector search via bundled nomic-embed-code embeddings compiled into the binary — no API key, no Ollama, no Docker. Ships as a single static C binary (zero runtime dependencies) for macOS, Linux, and Windows. All processing is local — the embeddings run on-device; there is no embedded LLM and no API key. +> An open-source Model Context Protocol (MCP) server that indexes a codebase into a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. AI coding agents query the graph instead of reading files one by one, answering structural questions with roughly 120x fewer tokens (~3,400 vs ~412,000 across five structural queries). Parses 158 languages via vendored tree-sitter grammars, with Hybrid LSP semantic type resolution for Python, TypeScript/JavaScript, PHP, C#, Go, C/C++, Java, Kotlin, and Rust. It also generates semantic graph edges (SEMANTICALLY_RELATED for vocabulary-mismatch matches, SIMILAR_TO for near-clone detection) and supports semantic vector search via bundled nomic-embed-code embeddings compiled into the native executable — no API key, no Ollama, no Docker. Ships a native executable with an authenticated integration asset; the UI variant adds one content-addressed frontend pack. All processing is local — the embeddings run on-device; there is no embedded LLM and no API key. ## Key facts @@ -15,7 +15,7 @@ - Supported agents: 43 automatic/conditional client surfaces (37 automatically detected + 6 conditional/explicit): Claude Code, Codex CLI, Gemini CLI, Zed, OpenCode, Antigravity, Aider, KiloCode, VS Code, Cursor, Windsurf, Augment / Auggie, OpenClaw, Kiro, Junie, Hermes, OpenHands, Cline, Warp, Qwen Code, GitHub Copilot CLI, Factory Droid, Crush, Goose, Mistral Vibe, Qoder CLI, Kimi Code CLI, GitLab Duo CLI, Rovo Dev CLI, Amp, Devin CLI / Local, Tabnine, Continue / cn, Visual Studio, TRAE, Roo Code, Amazon Q Developer IDE, CodeBuddy Code CLI, IBM Bob IDE, IBM Bob Shell, Pochi, Pi, and Sourcegraph Cody. - Agent profiles: documented custom-agent formats receive Scout (fast/provisional), Verify (default/task-directed), and Auditor (bounded/full verification) definitions. Every direct tier checks exact path/scope coverage with check_index_coverage and falls back to source for flagged gaps; unsafe child-MCP formats use explicit parent handoff. Kiro and Junie use positive-allowlist Scout/Analysis server profiles (7/11 tools); Qoder combines named-server selection with exact tier-specific MCP tool IDs, while Factory uses exact registered IDs without additive whole-server exposure. Foreign Junie aliases are preserved and force parent handoff. - Performance: Linux kernel (28M LOC, 75K files) full index in 3 minutes → 4.81M nodes, 7.72M edges; Cypher queries in under 1ms. -- Distribution: single static C binary; also npm, PyPI, Homebrew, Scoop, Winget, Chocolatey, AUR, and `go install`. +- Distribution: verified runtime set (native executable plus authenticated release-owned assets); fully static Linux builds are available in the `-portable` archives. Also available through npm, PyPI, Homebrew, Scoop, Winget, Chocolatey, AUR, and `go install`. ## Links diff --git a/install.ps1 b/install.ps1 index a65a1a342d..a434e78fbc 100644 --- a/install.ps1 +++ b/install.ps1 @@ -21,6 +21,7 @@ $WindowsArchiveNames = @( "install.ps1", "THIRD_PARTY_NOTICES.md" ) +$UiPackPattern = '^cbm-ui-[0-9a-f]{64}\.pack$' $BaseUrl = if ($env:CBM_DOWNLOAD_URL) { $env:CBM_DOWNLOAD_URL } else { "https://github.com/$Repo/releases/latest/download" } try { $BaseUri = [Uri]$BaseUrl } catch { $BaseUri = $null } @@ -88,6 +89,50 @@ function Invoke-CbmDownload { } } +function New-CbmExclusiveSiblingTemp { + param([Parameter(Mandatory=$true)][string]$Destination) + + $directory = [System.IO.Path]::GetDirectoryName($Destination) + $leaf = [System.IO.Path]::GetFileName($Destination) + for ($attempt = 0; $attempt -lt 32; $attempt++) { + $random = [System.IO.Path]::GetRandomFileName() + $candidate = Join-Path $directory ".$leaf.tmp-$random" + try { + $reservation = [System.IO.File]::Open( + $candidate, + [System.IO.FileMode]::CreateNew, + [System.IO.FileAccess]::Write, + [System.IO.FileShare]::None) + $reservation.Dispose() + return $candidate + } catch [System.IO.IOException] { + # A collision belongs to someone else. Never remove it; choose a + # fresh unpredictable sibling and reserve that path exclusively. + } + } + throw "could not reserve an exclusive temporary sibling for $Destination" +} + +function New-CbmExclusiveTempDirectory { + param([Parameter(Mandatory=$true)][string]$ParentDirectory) + + for ($attempt = 0; $attempt -lt 32; $attempt++) { + $candidate = Join-Path $ParentDirectory ( + "cbm-install-" + [guid]::NewGuid().ToString("N") + ) + try { + # Without -Force, an existing path is never adopted. Only return a + # directory successfully created by this installer invocation. + New-Item -ItemType Directory -Path $candidate -ErrorAction Stop | Out-Null + return $candidate + } catch [System.IO.IOException] { + # A collision belongs to someone else. Never remove it; choose a + # fresh unpredictable name and try again within the fixed bound. + } + } + throw "could not reserve an exclusive installer temporary directory" +} + # Detect variant from args (--ui or --standard) $Variant = "standard" $SkipConfig = $false @@ -135,8 +180,7 @@ if ($Variant -eq "ui") { $Url = "$BaseUrl/$Archive" # Download -$TmpDir = Join-Path ([System.IO.Path]::GetTempPath()) "cbm-install-$(Get-Random)" -New-Item -ItemType Directory -Path $TmpDir -Force | Out-Null +$TmpDir = New-CbmExclusiveTempDirectory -ParentDirectory ([System.IO.Path]::GetTempPath()) Write-Host "Downloading $Archive..." try { @@ -189,7 +233,7 @@ try { # Validate the zip namespace before extraction. Windows paths are # case-insensitive, so two entries that differ only in case are ambiguous and # must never be allowed to overwrite each other. The official five entries are -# required at the archive root with their exact release names. +# required at the archive root; UI adds exactly one hash-shaped pack. try { Add-Type -AssemblyName System.IO.Compression.FileSystem $zip = [System.IO.Compression.ZipFile]::OpenRead("$TmpDir\$Archive") @@ -197,6 +241,8 @@ try { $seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase) $archiveCounts = @{} foreach ($archiveName in $WindowsArchiveNames) { $archiveCounts[$archiveName] = 0 } + $uiPackName = $null + $uiPackCount = 0 foreach ($entry in $zip.Entries) { $entryName = $entry.FullName.Replace('\', '/') $isDirectory = $entryName.EndsWith('/') @@ -214,18 +260,28 @@ try { if (-not $seen.Add($pathForSegments)) { throw "duplicate or case-conflicting zip entry: $($entry.FullName)" } - if (-not ($WindowsArchiveNames -ccontains $entryName) -or $isDirectory) { + if ($isDirectory) { + throw "archive contains an unexpected root entry: $($entry.FullName)" + } + if ($WindowsArchiveNames -ccontains $entryName) { + $archiveCounts[$entryName] = $archiveCounts[$entryName] + 1 + } elseif ($Variant -eq "ui" -and $entryName -cmatch $UiPackPattern) { + $uiPackName = $entryName + $uiPackCount++ + } else { throw "archive contains an unexpected root entry: $($entry.FullName)" } - $archiveCounts[$entryName] = $archiveCounts[$entryName] + 1 } foreach ($archiveName in $WindowsArchiveNames) { if ($archiveCounts[$archiveName] -ne 1) { throw "archive must contain exactly one $archiveName" } } - if ($seen.Count -ne $WindowsArchiveNames.Count) { - throw "archive does not match the exact Windows release allowlist" + $expectedUiPackCount = if ($Variant -eq "ui") { 1 } else { 0 } + $expectedArchiveCount = $WindowsArchiveNames.Count + $expectedUiPackCount + if ($uiPackCount -ne $expectedUiPackCount -or + $seen.Count -ne $expectedArchiveCount) { + throw "archive does not match the exact $Variant Windows release allowlist" } } finally { $zip.Dispose() @@ -242,6 +298,21 @@ try { Write-Host "Extracting..." Expand-Archive -Path "$TmpDir\$Archive" -DestinationPath $TmpDir -Force +foreach ($archiveName in $WindowsArchiveNames) { + $extractedMember = Join-Path $TmpDir $archiveName + if (-not (Test-Path -LiteralPath $extractedMember -PathType Leaf)) { + Write-Host "error: release member is not a regular file: $archiveName" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 + } + $extractedItem = Get-Item -LiteralPath $extractedMember + if ($extractedItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + Write-Host "error: refusing reparse-point release member: $archiveName" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 + } +} + $DownloadedBinary = Join-Path $TmpDir $BinName if (-not (Test-Path -LiteralPath $DownloadedBinary -PathType Leaf)) { Write-Host "error: $BinName not found after extraction" -ForegroundColor Red @@ -255,6 +326,29 @@ if ($binaryItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { exit 1 } +$DownloadedUiPack = $null +if ($Variant -eq "ui") { + $DownloadedUiPack = Join-Path $TmpDir $uiPackName + if (-not (Test-Path -LiteralPath $DownloadedUiPack -PathType Leaf)) { + Write-Host "error: UI asset pack not found after extraction" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 + } + $packItem = Get-Item -LiteralPath $DownloadedUiPack + if ($packItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) { + Write-Host "error: refusing reparse-point UI asset pack" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 + } + $expectedPackDigest = $uiPackName.Substring(7, 64) + $actualPackDigest = (Get-FileHash -LiteralPath $DownloadedUiPack -Algorithm SHA256).Hash.ToLower() + if ($expectedPackDigest -cne $actualPackDigest) { + Write-Host "error: UI asset pack digest does not match its filename" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 + } +} + # Prove the downloaded binary runs before touching an existing installation. try { $candidateVersion = & $DownloadedBinary --version 2>&1 @@ -266,32 +360,12 @@ try { exit 1 } +# The candidate publishes the runtime set under one native activation guard, +# with sidecars before the executable and retained per-file backups for +# cooperative rollback. This is recoverable/fail-closed ordering, not a claim +# that several filesystem entries change in one crash-atomic transaction. $Dest = Join-Path $InstallDir $BinName -# Retire the running installation before replacing it. Windows keeps an image -# lock on a running .exe: the file cannot be overwritten, but it CAN be renamed -# out of the way, which is what makes an in-place update possible from here. -if (Test-Path -LiteralPath $Dest -PathType Leaf) { - try { & $Dest daemon stop 2>&1 | Out-Null } catch { } - $retired = "$Dest.retired-$(Get-Date -Format yyyyMMddHHmmss)" - $renamed = $false - foreach ($attempt in 1..10) { - try { Move-Item -LiteralPath $Dest -Destination $retired -Force -ErrorAction Stop; $renamed = $true; break } - catch { Start-Sleep -Milliseconds 500 } - } - if (-not $renamed) { - Write-Host "error: could not retire the existing $BinName - close all running" -ForegroundColor Red - Write-Host " codebase-memory-mcp sessions and coding agents, then re-run." -ForegroundColor Red - Remove-Item -Recurse -Force $TmpDir -ErrorAction SilentlyContinue - exit 1 - } - # A retired image stays locked until its last process exits; delete it when - # we can, and leave it for the next run when we cannot. Never fail here. - Remove-Item -LiteralPath $retired -Force -ErrorAction SilentlyContinue -} -Get-ChildItem -LiteralPath $InstallDir -Filter "$BinName.retired-*" -ErrorAction SilentlyContinue | - ForEach-Object { Remove-Item -LiteralPath $_.FullName -Force -ErrorAction SilentlyContinue } - $InstallArgs = @("install", "-y", "--force", "--dir=$InstallDir") if ($SkipConfig) { $InstallArgs += "--skip-config" } & $DownloadedBinary @InstallArgs @@ -312,35 +386,21 @@ if ($LASTEXITCODE -ne 0) { $DownloadedInstaller = Join-Path $TmpDir "install.ps1" if (Test-Path -LiteralPath $DownloadedInstaller -PathType Leaf) { $InstallerDest = Join-Path $InstallDir "install.ps1" - $InstallerTmp = "$InstallerDest.new" + $InstallerTmp = $null try { + $InstallerTmp = New-CbmExclusiveSiblingTemp -Destination $InstallerDest Copy-Item -LiteralPath $DownloadedInstaller -Destination $InstallerTmp -Force -ErrorAction Stop Move-Item -LiteralPath $InstallerTmp -Destination $InstallerDest -Force -ErrorAction Stop + $InstallerTmp = $null Write-Host "Installed updater -> $InstallerDest" } catch { - Remove-Item -LiteralPath $InstallerTmp -Force -ErrorAction SilentlyContinue + if ($InstallerTmp) { + Remove-Item -LiteralPath $InstallerTmp -Force -ErrorAction SilentlyContinue + } Write-Host "note: could not place install.ps1 in $InstallDir (update will explain where to find it)" } } -# Place the integration-template asset beside the installed binary, mirroring -# install.sh. `install` above published a verified copy to ~/.cbm/assets, but a -# later install/uninstall run from $InstallDir resolves the asset next to the -# binary first; without this copy that lookup misses and the run fails closed -# with "integration assets missing". -$DownloadedAsset = Join-Path $TmpDir "cbm-integrations.json" -if (Test-Path -LiteralPath $DownloadedAsset -PathType Leaf) { - $AssetDest = Join-Path $InstallDir "cbm-integrations.json" - $AssetTmp = "$AssetDest.new" - try { - Copy-Item -LiteralPath $DownloadedAsset -Destination $AssetTmp -Force -ErrorAction Stop - Move-Item -LiteralPath $AssetTmp -Destination $AssetDest -Force -ErrorAction Stop - } catch { - Remove-Item -LiteralPath $AssetTmp -Force -ErrorAction SilentlyContinue - Write-Host "note: could not place cbm-integrations.json in $InstallDir (asset resolves from ~/.cbm/assets)" - } -} - # Verify try { $ver = & $Dest --version 2>&1 diff --git a/install.sh b/install.sh index e0551055ed..b9e9cea3f0 100755 --- a/install.sh +++ b/install.sh @@ -231,6 +231,71 @@ if [ "$EXPECTED" != "$ACTUAL" ]; then fi echo "Checksum verified." +# Validate the complete archive namespace before extraction. Standard releases +# are the canonical five files; UI releases add exactly one root-level, +# content-addressed pack. Anything else is a release-integrity failure, not a +# sidecar to ignore. +if [ "$OS" = "windows" ]; then + ARCHIVE_BINARY="codebase-memory-mcp.exe" + ARCHIVE_INSTALLER="install.ps1" +else + ARCHIVE_BINARY="codebase-memory-mcp" + ARCHIVE_INSTALLER="install.sh" +fi +ARCHIVE_MEMBERS_FILE="$DLDIR/archive-members.txt" +if [ "$EXT" = "zip" ]; then + if ! unzip -Z1 "$DLDIR/$ARCHIVE" > "$ARCHIVE_MEMBERS_FILE"; then + echo "error: could not enumerate release archive" >&2 + exit 1 + fi +else + if ! tar -tzf "$DLDIR/$ARCHIVE" > "$ARCHIVE_MEMBERS_FILE"; then + echo "error: could not enumerate release archive" >&2 + exit 1 + fi +fi + +BINARY_MEMBERS=0 +INTEGRATION_MEMBERS=0 +LICENSE_MEMBERS=0 +INSTALLER_MEMBERS=0 +NOTICE_MEMBERS=0 +UI_PACK_MEMBERS=0 +ARCHIVE_MEMBER_COUNT=0 +UI_PACK_NAME="" +while IFS= read -r member || [ -n "$member" ]; do + ARCHIVE_MEMBER_COUNT=$((ARCHIVE_MEMBER_COUNT + 1)) + case "$member" in + "$ARCHIVE_BINARY") BINARY_MEMBERS=$((BINARY_MEMBERS + 1)) ;; + cbm-integrations.json) INTEGRATION_MEMBERS=$((INTEGRATION_MEMBERS + 1)) ;; + LICENSE) LICENSE_MEMBERS=$((LICENSE_MEMBERS + 1)) ;; + "$ARCHIVE_INSTALLER") INSTALLER_MEMBERS=$((INSTALLER_MEMBERS + 1)) ;; + THIRD_PARTY_NOTICES.md) NOTICE_MEMBERS=$((NOTICE_MEMBERS + 1)) ;; + *) + if [ "$VARIANT" = "ui" ] && + [[ "$member" =~ ^cbm-ui-[0-9a-f]{64}\.pack$ ]]; then + UI_PACK_MEMBERS=$((UI_PACK_MEMBERS + 1)) + UI_PACK_NAME="$member" + else + echo "error: release archive contains unexpected member: $member" >&2 + exit 1 + fi + ;; + esac +done < "$ARCHIVE_MEMBERS_FILE" + +EXPECTED_MEMBER_COUNT=5 +if [ "$VARIANT" = "ui" ]; then + EXPECTED_MEMBER_COUNT=6 +fi +if [ "$BINARY_MEMBERS" -ne 1 ] || [ "$INTEGRATION_MEMBERS" -ne 1 ] || + [ "$LICENSE_MEMBERS" -ne 1 ] || [ "$INSTALLER_MEMBERS" -ne 1 ] || + [ "$NOTICE_MEMBERS" -ne 1 ] || [ "$UI_PACK_MEMBERS" -ne $((EXPECTED_MEMBER_COUNT - 5)) ] || + [ "$ARCHIVE_MEMBER_COUNT" -ne "$EXPECTED_MEMBER_COUNT" ]; then + echo "error: release archive does not match the exact $VARIANT member set" >&2 + exit 1 +fi + # Extract echo "Extracting..." if [ "$EXT" = "zip" ]; then @@ -239,12 +304,41 @@ else tar -xzf "$DLDIR/$ARCHIVE" -C "$DLDIR" fi -DLBIN="$DLDIR/codebase-memory-mcp" -if [ ! -f "$DLBIN" ]; then +for extracted_member in "$ARCHIVE_BINARY" cbm-integrations.json LICENSE \ + "$ARCHIVE_INSTALLER" THIRD_PARTY_NOTICES.md; do + if [ ! -f "$DLDIR/$extracted_member" ] || [ -L "$DLDIR/$extracted_member" ]; then + echo "error: release member is not a regular file: $extracted_member" >&2 + exit 1 + fi +done + +DLBIN="$DLDIR/$ARCHIVE_BINARY" +if [ ! -f "$DLBIN" ] || [ -L "$DLBIN" ]; then echo "error: binary not found after extraction" >&2 exit 1 fi +DL_UI_PACK="" +if [ "$VARIANT" = "ui" ]; then + DL_UI_PACK="$DLDIR/$UI_PACK_NAME" + if [ ! -f "$DL_UI_PACK" ] || [ -L "$DL_UI_PACK" ]; then + echo "error: UI asset pack not found after extraction" >&2 + exit 1 + fi + PACK_EXPECTED="${UI_PACK_NAME#cbm-ui-}" + PACK_EXPECTED="${PACK_EXPECTED%.pack}" + if command -v sha256sum &>/dev/null; then + PACK_ACTUAL=$(sha256sum "$DL_UI_PACK" | awk '{print $1}') + else + PACK_ACTUAL=$(shasum -a 256 "$DL_UI_PACK" | awk '{print $1}') + fi + PACK_ACTUAL=$(printf '%s' "$PACK_ACTUAL" | tr 'A-F' 'a-f') + if [ "$PACK_EXPECTED" != "$PACK_ACTUAL" ]; then + echo "error: UI asset pack digest does not match its filename" >&2 + exit 1 + fi +fi + # macOS: fix signing if [ "$OS" = "darwin" ]; then echo "Fixing macOS code signing..." @@ -261,7 +355,20 @@ if ! CANDIDATE_VERSION=$("$DLBIN" --version 2>&1); then fi echo "Verified candidate: $CANDIDATE_VERSION" +# The candidate publishes the runtime set under one native activation guard, +# with sidecars before the executable and retained per-file backups for +# cooperative rollback. The set is intentionally recoverable/fail-closed, not +# claimed to be a crash-atomic multi-file filesystem transaction. +new_exclusive_sibling_temp() { + local destination="$1" + local directory basename + directory="$(dirname "$destination")" + basename="$(basename "$destination")" + mktemp "$directory/.${basename}.tmp.XXXXXX" +} + DEST="$INSTALL_DIR/codebase-memory-mcp" +[ "$OS" = "windows" ] && DEST="$INSTALL_DIR/codebase-memory-mcp.exe" INSTALL_ARGS=(-y --force "--dir=$INSTALL_DIR") if [ "$SKIP_CONFIG" = true ]; then INSTALL_ARGS+=(--skip-config) @@ -283,36 +390,20 @@ fi # working install, and `update` falls back to explaining where to find it. DL_INSTALLER="$DLDIR/install.sh" if [ -f "$DL_INSTALLER" ]; then - INSTALLER_TMP="$INSTALL_DIR/.install.sh.$$" - if cp "$DL_INSTALLER" "$INSTALLER_TMP" 2>/dev/null && + INSTALLER_TMP="" + if INSTALLER_TMP="$(new_exclusive_sibling_temp "$INSTALL_DIR/install.sh" 2>/dev/null)" && + cp "$DL_INSTALLER" "$INSTALLER_TMP" 2>/dev/null && chmod 755 "$INSTALLER_TMP" 2>/dev/null && mv -f "$INSTALLER_TMP" "$INSTALL_DIR/install.sh" 2>/dev/null; then echo "Installed updater -> $INSTALL_DIR/install.sh" else - rm -f "$INSTALLER_TMP" 2>/dev/null || true + # A non-empty path was exclusively created by mktemp above. If + # reservation failed, leave every pre-existing sibling untouched. + [ -z "$INSTALLER_TMP" ] || rm -f "$INSTALLER_TMP" 2>/dev/null || true echo "note: could not place install.sh in $INSTALL_DIR (update will explain where to find it)" fi fi -# Place the integration-template asset beside the installed binary. `install` -# above already published a verified copy to ~/.cbm/assets//, but a -# later `install`/`uninstall` run from INSTALL_DIR resolves the asset next to -# the binary FIRST — and without this copy that lookup misses, so a re-run or an -# uninstall would fail closed with "integration assets missing" even though a -# valid install just completed. Atomic rename, best effort, same rationale as -# the installer copy above. -DL_ASSET="$DLDIR/cbm-integrations.json" -if [ -f "$DL_ASSET" ]; then - ASSET_TMP="$INSTALL_DIR/.cbm-integrations.json.$$" - if cp "$DL_ASSET" "$ASSET_TMP" 2>/dev/null && - mv -f "$ASSET_TMP" "$INSTALL_DIR/cbm-integrations.json" 2>/dev/null; then - : - else - rm -f "$ASSET_TMP" 2>/dev/null || true - echo "note: could not place cbm-integrations.json in $INSTALL_DIR (asset resolves from ~/.cbm/assets)" - fi -fi - # Verify VERSION=$("$DEST" --version 2>&1) || { echo "error: installed binary failed to run" >&2 diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index 5d6309c7ef..ea149dd9c5 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -707,6 +707,10 @@ const char *cbm_index_quarantine_phase(const char *rel_path) { return (const char *)cbm_ht_get(g_quarantine_set, rel_path); } +#ifdef CBM_ENABLE_TEST_SEAMS +/* Deterministic supervisor fault injection belongs only in explicitly + * seam-enabled test artifacts. Ordinary release binaries must never expose a + * filename-selected abort or infinite loop through environment variables. */ static void cbm_test_fault_inject(const char *rel_path) { if (!rel_path || !rel_path[0]) { return; @@ -722,6 +726,7 @@ static void cbm_test_fault_inject(const char *rel_path) { } } } +#endif /* Pre-parse nesting guard for pathologically nested input. tree-sitter's GLR * parser recurses once per nesting level inside stack_node_add_link @@ -1148,7 +1153,9 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua } cbm_index_mark_start(rel_path); +#ifdef CBM_ENABLE_TEST_SEAMS cbm_test_fault_inject(rel_path); +#endif // Get language spec const CBMLangSpec *spec = cbm_lang_spec(language); diff --git a/pkg/aur/.SRCINFO b/pkg/aur/.SRCINFO index 14bf16fe51..79ae675d9b 100644 --- a/pkg/aur/.SRCINFO +++ b/pkg/aur/.SRCINFO @@ -1,5 +1,5 @@ pkgbase = codebase-memory-mcp - pkgdesc = Fast code intelligence engine for AI coding agents — single static binary MCP server + pkgdesc = Fast code intelligence engine — native executable with verified integration asset pkgver = 0.8.1 pkgrel = 1 url = https://github.com/DeusData/codebase-memory-mcp diff --git a/pkg/aur/PKGBUILD b/pkg/aur/PKGBUILD index 957c682b18..a28f6a7d90 100644 --- a/pkg/aur/PKGBUILD +++ b/pkg/aur/PKGBUILD @@ -2,7 +2,7 @@ pkgname=codebase-memory-mcp pkgver=0.8.1 pkgrel=1 -pkgdesc="Fast code intelligence engine for AI coding agents — single static binary MCP server" +pkgdesc="Fast code intelligence engine — native executable with verified integration asset" arch=('x86_64' 'aarch64') url="https://github.com/DeusData/codebase-memory-mcp" license=('MIT') @@ -18,6 +18,8 @@ sha256sums_aarch64=('d2f842d1365da5c35d9c5796f57a821c9745267350994346735e1e6e04d package() { install -Dm755 "${srcdir}/codebase-memory-mcp" \ "${pkgdir}/usr/bin/codebase-memory-mcp" + install -Dm644 "${srcdir}/cbm-integrations.json" \ + "${pkgdir}/usr/share/codebase-memory-mcp/cbm-integrations.json" install -Dm644 "${srcdir}/LICENSE" \ "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" # Third-party attribution bundle (present in archives since v0.8.1) diff --git a/pkg/chocolatey/codebase-memory-mcp.nuspec b/pkg/chocolatey/codebase-memory-mcp.nuspec index 5175d96ead..0c8f3f1775 100644 --- a/pkg/chocolatey/codebase-memory-mcp.nuspec +++ b/pkg/chocolatey/codebase-memory-mcp.nuspec @@ -13,13 +13,14 @@ mcp ai claude code-intelligence codebase developer-tools tree-sitter llm

Fast code intelligence engine for AI coding agents -codebase-memory-mcp is a single static binary MCP server that indexes codebases at -extreme speed and exposes 14 MCP tools for AI coding agents. +codebase-memory-mcp is a native MCP server distributed with authenticated +release-owned runtime assets. Chocolatey retains the complete extracted runtime +set and exposes its executable through a PATH shim. Full indexes an average repository in milliseconds, the Linux kernel (28M LOC) in 3 minutes. Answers structural queries in under 1ms. Supports 159 languages via -vendored tree-sitter grammars. Zero dependencies. Plug and play across 11 coding -agents including Claude Code, Codex CLI, Gemini CLI, and Zed. +vendored tree-sitter grammars. No hosted service or API key. Plug and play across +11 coding agents including Claude Code, Codex CLI, Gemini CLI, and Zed. After installation, run: diff --git a/pkg/chocolatey/tools/chocolateyInstall.ps1 b/pkg/chocolatey/tools/chocolateyInstall.ps1 index 191ce7f83c..3c83ee8c19 100644 --- a/pkg/chocolatey/tools/chocolateyInstall.ps1 +++ b/pkg/chocolatey/tools/chocolateyInstall.ps1 @@ -16,10 +16,3 @@ Install-ChocolateyZipPackage ` # Shim the binary so it is on PATH $binPath = Join-Path $installDir 'codebase-memory-mcp.exe' Install-BinFile -Name 'codebase-memory-mcp' -Path $binPath - -# Configure coding agents (non-fatal) -try { - & $binPath install -y 2>&1 | Out-Null -} catch { - Write-Warning "Agent configuration failed (non-fatal). Run manually: codebase-memory-mcp install" -} diff --git a/pkg/glama/Dockerfile b/pkg/glama/Dockerfile index 2b06f7f585..23f0fa7020 100644 --- a/pkg/glama/Dockerfile +++ b/pkg/glama/Dockerfile @@ -15,12 +15,13 @@ ARG TARGETARCH RUN apt-get update \ && apt-get install -y --no-install-recommends curl ca-certificates \ && curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/latest/download/codebase-memory-mcp-linux-${TARGETARCH}-portable.tar.gz" \ - | tar -xz -C /tmp codebase-memory-mcp LICENSE THIRD_PARTY_NOTICES.md \ + | tar -xz -C /tmp codebase-memory-mcp cbm-integrations.json LICENSE THIRD_PARTY_NOTICES.md \ && chmod +x /tmp/codebase-memory-mcp \ && rm -rf /var/lib/apt/lists/* FROM debian:bookworm-slim COPY --from=fetch /tmp/codebase-memory-mcp /usr/local/bin/codebase-memory-mcp +COPY --from=fetch /tmp/cbm-integrations.json /usr/local/share/codebase-memory-mcp/cbm-integrations.json COPY --from=fetch /tmp/LICENSE /tmp/THIRD_PARTY_NOTICES.md /usr/share/doc/codebase-memory-mcp/ ENV CBM_CACHE_DIR=/tmp/cbm ENTRYPOINT ["codebase-memory-mcp"] diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 5677a79093..ffc8efdfdc 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -17,6 +17,8 @@ import ( "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" + "errors" "fmt" "io" "net" @@ -26,16 +28,17 @@ import ( "os/exec" "path/filepath" "runtime" + "sort" "strings" "syscall" "time" ) const ( - repo = "DeusData/codebase-memory-mcp" - version = "0.8.1" - windowsLauncherName = "codebase-memory-mcp.exe" - windowsPayloadName = "codebase-memory-mcp.payload.exe" + repo = "DeusData/codebase-memory-mcp" + version = "0.8.1" + windowsBinaryName = "codebase-memory-mcp.exe" + integrationsFileName = "cbm-integrations.json" maxRedirects = 5 requestTimeout = 2 * time.Minute @@ -43,30 +46,101 @@ const ( responseHeaderTimeout = 30 * time.Second candidateTimeout = 15 * time.Second maxChecksumManifestSize = 1024 * 1024 + maxReleaseArchiveSize = int64(256 * 1024 * 1024) + maxArchiveMembers = 64 + maxArchiveMemberSize = int64(256 * 1024 * 1024) + maxArchiveExpandedSize = int64(512 * 1024 * 1024) - windowsPairLockName = ".codebase-memory-mcp-pair.lock" - windowsPairLockWait = 45 * time.Second - windowsPairOwnerlessStale = 30 * time.Second - windowsPairLockPoll = 25 * time.Millisecond - windowsPairLockOwnerFile = "owner" - windowsPairTransactionTagSize = 16 + runtimeSetLockName = ".codebase-memory-mcp-runtime.lock" + runtimeSetLockWait = 45 * time.Second + runtimeSetOwnerlessStale = 30 * time.Second + runtimeSetLockLease = 5 * time.Minute + runtimeSetLockHeartbeat = 20 * time.Second + runtimeSetLegacyStale = time.Hour + runtimeSetLockPoll = 25 * time.Millisecond + runtimeSetLockOwnerFile = "owner.json" + runtimeSetLockTokenSize = 16 + runtimeBackupPrefix = ".cbm-runtime-backup-" + runtimeBackupRetired = ".retirement-complete" + runtimeBackupCleanupOnly = ".cleanup-only" + runtimeBackupBeforeMarkerEvent = "before-retirement-marker" + runtimeBackupCleanupRemovedEvent = "cleanup-member-removed" ) -type windowsPairLock struct { +type archiveResourceLimits struct { + compressedBytes int64 + members int + memberBytes int64 + expandedBytes int64 +} + +var defaultArchiveResourceLimits = archiveResourceLimits{ + compressedBytes: maxReleaseArchiveSize, + members: maxArchiveMembers, + memberBytes: maxArchiveMemberSize, + expandedBytes: maxArchiveExpandedSize, +} + +type runtimeSetLock struct { path string token string + file *os.File +} + +type runtimeSetLockOwnerRecord struct { + PID int `json:"pid"` + Token string `json:"token"` + LeaseExpires int64 `json:"lease_expires_ms,omitempty"` +} + +type runtimeBackupFile struct { + path string + digest [sha256.Size]byte } +type runtimeBackupTransaction struct { + path string + retired bool + cleanupOnly bool + files map[string]runtimeBackupFile +} + +// Tests use this observer to distinguish a publisher waiting on the runtime +// lock from one entering the shared publication sequence. Production leaves +// it nil. +var ( + runtimeSetLockWaitObserver func() + runtimeSetLockClaimObserver func() error + runtimeBackupCrashObserver func(string, string) error + runtimeSetProcessAlive = platformRuntimeSetLockProcessAlive + runtimeMutationSnapshotCleanup = os.RemoveAll +) + func main() { + mutation := runtimeMutationAction(os.Args[1:]) + if mutation == "update" { + fmt.Fprintln( + os.Stderr, + "This Go wrapper is maintained by Go. Update it with \"go install github.com/DeusData/codebase-memory-mcp/pkg/go/cmd/codebase-memory-mcp@latest\".", + ) + os.Exit(2) + } executable, err := ensureBinary() if err != nil { fmt.Fprintf(os.Stderr, "codebase-memory-mcp: %v\n", err) os.Exit(1) } - if err := execBinary(executable, os.Args[1:]); err != nil { + args := nativeArgs(os.Args[1:]) + if mutation == "install" || mutation == "uninstall" { + err = execBinaryWithRuntimeLock(executable, args) + } else { + err = execBinary(executable, args) + } + if err != nil { if runtime.GOOS == "windows" { - if exitErr, ok := err.(*exec.ExitError); ok { - printPortableMutationGuidance(os.Args[1:]) + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + printPortableMutationGuidance(args) os.Exit(exitErr.ExitCode()) } } @@ -76,78 +150,161 @@ func main() { } func ensureBinary() (string, error) { - payload := binPath() - ready := regularFileExists(payload) - if runtime.GOOS == "windows" { - ready = ready && regularFileExists(windowsLauncherPath()) - if ready { - ready = verifyCandidate(windowsLauncherPath()) == nil - } + binary := binPath() + ready, err := runtimeSetReadyLocked( + filepath.Dir(binary), filepath.Base(binary), runtimeVariant(), verifyCandidate, + ) + if err != nil { + return "", err } if ready { - return executionPathForOS(payload, runtime.GOOS), nil + return executionPathForOS(binary, runtime.GOOS), nil } - if err := download(payload); err != nil { + if err := download(binary); err != nil { return "", err } - return executionPathForOS(payload, runtime.GOOS), nil + return executionPathForOS(binary, runtime.GOOS), nil } func binPath() string { - name := "codebase-memory-mcp" - if runtime.GOOS == "windows" { - // The payload remains the immutable exact-version pair's readiness - // signal. executionPathForOS selects its adjacent permanent launcher. - name = windowsPayloadName - } - return filepath.Join(cacheDir(), version, name) + return filepath.Join( + cacheDir(), version, runtimeVariant(), binaryNameForOS(runtime.GOOS), + ) } -func executionPathForOS(payload, targetOS string) string { +func binaryNameForOS(targetOS string) string { if targetOS == "windows" { - return filepath.Join(filepath.Dir(payload), windowsLauncherName) + return windowsBinaryName } - return payload + return "codebase-memory-mcp" } -func windowsLauncherPath() string { - return filepath.Join(filepath.Dir(binPath()), windowsLauncherName) +func executionPathForOS(binary, targetOS string) string { + return binary } -func regularFileExists(path string) bool { - info, err := os.Stat(path) - return err == nil && info.Mode().IsRegular() +func runtimeVariant() string { + if strings.EqualFold(os.Getenv("CBM_VARIANT"), "ui") { + return "ui" + } + return "standard" } func printPortableMutationGuidance(args []string) { - action := portableMutationAction(args) - if action == "" { + action := runtimeMutationAction(args) + if action != "uninstall" { return } - packageCommand := "go install github.com/DeusData/codebase-memory-mcp/pkg/go/cmd/codebase-memory-mcp@latest" - if action == "uninstall" { - packageCommand = "Remove-Item (Get-Command codebase-memory-mcp).Source" - } + packageCommand := "Remove-Item (Get-Command codebase-memory-mcp).Source" fmt.Fprintf( os.Stderr, - "This Go Windows copy is portable. Use %q for package maintenance, or run %q once to create a managed launcher with coordinated self-update/uninstall.\n", + "This Go Windows copy is portable. Use %q for package maintenance, or run %q once to create a managed installation with coordinated self-update/uninstall.\n", packageCommand, "codebase-memory-mcp install --yes", ) } -func portableMutationAction(args []string) string { +func runtimeMutationAction(args []string) string { for _, argument := range args { switch argument { - case "cli", "hook-augment", "config", "install", "--help", "-h", "--version": + case "--help", "-h", "--version": + return "" + case "cli", "hook-augment", "config": return "" - case "update", "uninstall": + case "install", "update", "uninstall": return argument } + if strings.HasPrefix(argument, "-") { + continue + } + return "" } return "" } +func defaultManagedInstallDir() string { + if runtime.GOOS == "windows" { + base := os.Getenv("LOCALAPPDATA") + if base == "" { + if home, err := os.UserHomeDir(); err == nil { + base = filepath.Join(home, "AppData", "Local") + } + } + return filepath.Join(base, "Programs", "codebase-memory-mcp") + } + if home, err := os.UserHomeDir(); err == nil { + return filepath.Join(home, ".local", "bin") + } + return filepath.Join(os.TempDir(), "codebase-memory-mcp-managed") +} + +func nativeArgs(args []string) []string { + result := append([]string(nil), args...) + if runtimeMutationAction(result) != "uninstall" { + return result + } + for _, argument := range result { + if argument == "--dir" || strings.HasPrefix(argument, "--dir=") { + return result + } + } + return append(result, "--dir", defaultManagedInstallDir()) +} + +func mutationTargetDirectory(args []string) (string, bool) { + for index, argument := range args { + if strings.HasPrefix(argument, "--dir=") { + return strings.TrimPrefix(argument, "--dir="), true + } + if argument == "--dir" && index+1 < len(args) { + return args[index+1], true + } + } + return "", false +} + +func pathsReferToSameDirectory(left, right string) bool { + leftAbsolute, leftErr := filepath.Abs(left) + rightAbsolute, rightErr := filepath.Abs(right) + if leftErr != nil || rightErr != nil { + return false + } + leftAbsolute = filepath.Clean(leftAbsolute) + rightAbsolute = filepath.Clean(rightAbsolute) + if leftAbsolute == rightAbsolute || + (runtime.GOOS == "windows" && strings.EqualFold(leftAbsolute, rightAbsolute)) { + return true + } + leftStatus, leftErr := os.Stat(leftAbsolute) + rightStatus, rightErr := os.Stat(rightAbsolute) + return leftErr == nil && rightErr == nil && + leftStatus.IsDir() && rightStatus.IsDir() && + os.SameFile(leftStatus, rightStatus) +} + +func mutationSnapshotEnvironment() []string { + blocked := []string{"CBM_ASSETS_DIR", "CBM_UI_ASSETS_DIR"} + result := make([]string, 0, len(os.Environ())) + for _, entry := range os.Environ() { + separator := strings.IndexByte(entry, '=') + name := entry + if separator >= 0 { + name = entry[:separator] + } + drop := false + for _, blockedName := range blocked { + if strings.EqualFold(name, blockedName) { + drop = true + break + } + } + if !drop { + result = append(result, entry) + } + } + return result +} + func cacheDir() string { if d := os.Getenv("CBM_CACHE_DIR"); d != "" { return d @@ -195,9 +352,24 @@ func goarch() string { } } +func archiveNamesForOS(platform, binaryName string) []string { + installer := "install.sh" + if platform == "windows" { + installer = "install.ps1" + } + return []string{ + binaryName, + integrationsFileName, + "LICENSE", + installer, + "THIRD_PARTY_NOTICES.md", + } +} + func download(dest string) error { platform := goos() arch := goarch() + selectedVariant := runtimeVariant() ext := "tar.gz" if platform == "windows" { ext = "zip" @@ -207,7 +379,14 @@ func download(dest string) error { if platform == "linux" { portable = "-portable" } - archive := fmt.Sprintf("codebase-memory-mcp-%s-%s%s.%s", platform, arch, portable, ext) + uiPrefix := "" + if selectedVariant == "ui" { + uiPrefix = "ui-" + } + archive := fmt.Sprintf( + "codebase-memory-mcp-%s%s-%s%s.%s", + uiPrefix, platform, arch, portable, ext, + ) url := fmt.Sprintf("https://github.com/%s/releases/download/v%s/%s", repo, version, archive) checksumURL := fmt.Sprintf("https://github.com/%s/releases/download/v%s/checksums.txt", repo, version) @@ -238,43 +417,37 @@ func download(dest string) error { return err } - binName := "codebase-memory-mcp" - requiredNames := []string{binName} - archiveNames := requiredNames - if platform == "windows" { - binName = windowsPayloadName - requiredNames = []string{windowsLauncherName, windowsPayloadName} - archiveNames = []string{ - windowsLauncherName, - windowsPayloadName, - "LICENSE", - "install.ps1", - "THIRD_PARTY_NOTICES.md", - } - } + binName := binaryNameForOS(platform) + archiveNames := archiveNamesForOS(platform, binName) + extractNames := []string{binName, integrationsFileName} + var runtimeNames []string if ext == "tar.gz" { - if err := extractTarGz(archivePath, tmp, binName); err != nil { + runtimeNames, err = extractTarGz( + archivePath, tmp, archiveNames, extractNames, selectedVariant, + ) + if err != nil { return fmt.Errorf("extraction failed: %w", err) } } else { - if err := extractZip(archivePath, tmp, archiveNames, requiredNames); err != nil { + runtimeNames, err = extractZip( + archivePath, tmp, archiveNames, extractNames, selectedVariant, + ) + if err != nil { return fmt.Errorf("extraction failed: %w", err) } } - for _, name := range requiredNames { + for _, name := range runtimeNames { extracted := filepath.Join(tmp, name) - if err := os.Chmod(extracted, 0755); err != nil { + mode := os.FileMode(0644) + if name == binName { + mode = 0755 + } + if err := os.Chmod(extracted, mode); err != nil { return fmt.Errorf("could not set candidate permissions for %s: %w", name, err) } } - if platform == "windows" { - // In portable mode the canonical launcher must successfully resolve the - // adjacent payload before the pair is cached. - if err := verifyCandidate(filepath.Join(tmp, windowsLauncherName)); err != nil { - return fmt.Errorf("downloaded launcher failed verification: %w", err) - } - } else if err := verifyCandidate(filepath.Join(tmp, binName)); err != nil { + if err := verifyCandidate(filepath.Join(tmp, binName)); err != nil { return err } @@ -282,12 +455,10 @@ func download(dest string) error { return fmt.Errorf("could not create cache dir: %w", err) } - if platform == "windows" { - if err := installWindowsPairAtomically(tmp, filepath.Dir(dest)); err != nil { - return fmt.Errorf("could not install Windows release pair: %w", err) - } - } else if err := installCandidateAtomically(filepath.Join(tmp, binName), dest); err != nil { - return fmt.Errorf("could not install binary: %w", err) + if err := publishRuntimeSetWithRecovery( + tmp, filepath.Dir(dest), binName, selectedVariant, verifyCandidate, + ); err != nil { + return fmt.Errorf("could not install runtime set: %w", err) } return nil @@ -325,9 +496,16 @@ var httpsOnlyClient = &http.Client{ } func httpGet(rawURL, dest string) error { + return httpGetWithLimit(rawURL, dest, maxReleaseArchiveSize) +} + +func httpGetWithLimit(rawURL, dest string, maxBytes int64) error { if err := validateURLScheme(rawURL); err != nil { return err } + if maxBytes <= 0 { + return fmt.Errorf("invalid compressed archive safety limit") + } resp, err := httpsOnlyClient.Get(rawURL) //nolint:gosec if err != nil { return err @@ -336,15 +514,31 @@ func httpGet(rawURL, dest string) error { if resp.StatusCode != http.StatusOK { return fmt.Errorf("HTTP %d for %s", resp.StatusCode, rawURL) } + if resp.ContentLength > maxBytes { + return fmt.Errorf( + "release archive exceeds the %d-byte compressed safety limit", + maxBytes, + ) + } f, err := os.Create(dest) if err != nil { return err } - _, copyErr := io.Copy(f, resp.Body) + written, copyErr := io.Copy(f, io.LimitReader(resp.Body, maxBytes+1)) closeErr := f.Close() + if copyErr == nil && written > maxBytes { + copyErr = fmt.Errorf( + "release archive exceeds the %d-byte compressed safety limit", + maxBytes, + ) + } if copyErr != nil { + _ = os.Remove(dest) return copyErr } + if closeErr != nil { + _ = os.Remove(dest) + } return closeErr } @@ -415,422 +609,1413 @@ func verifyChecksum(path, expected string) error { return nil } -func extractTarGz(archivePath, destDir, targetFile string) error { - f, err := os.Open(archivePath) - if err != nil { - return err - } - defer f.Close() - gz, err := gzip.NewReader(f) - if err != nil { - return err - } - defer gz.Close() - tr := tar.NewReader(gz) - found := false - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - name := strings.ReplaceAll(hdr.Name, "\\", "/") - if name == targetFile { - if found { - os.Remove(filepath.Join(destDir, targetFile)) - return fmt.Errorf("duplicate %s in archive", targetFile) - } - found = true - out, err := os.OpenFile( - filepath.Join(destDir, targetFile), - os.O_WRONLY|os.O_CREATE|os.O_EXCL, - 0600, - ) - if err != nil { - return err - } - _, copyErr := io.Copy(out, tr) //nolint:gosec - closeErr := out.Close() - if copyErr != nil { - os.Remove(filepath.Join(destDir, targetFile)) - return copyErr - } - if closeErr != nil { - os.Remove(filepath.Join(destDir, targetFile)) - return closeErr - } - } +func isUIPackName(name string) bool { + const prefix = "cbm-ui-" + const suffix = ".pack" + if len(name) != len(prefix)+sha256.Size*2+len(suffix) || + !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, suffix) { + return false } - if !found { - return fmt.Errorf("%s not found in archive", targetFile) + digest := strings.TrimSuffix(strings.TrimPrefix(name, prefix), suffix) + if digest != strings.ToLower(digest) { + return false } - return nil + decoded, err := hex.DecodeString(digest) + return err == nil && len(decoded) == sha256.Size } -func extractZip( - archivePath, destDir string, - archiveNames, extractNames []string, -) error { - r, err := zip.OpenReader(archivePath) - if err != nil { - return err - } - defer r.Close() - seen := make(map[string]struct{}, len(r.File)) +func validateArchiveMemberNames( + names, archiveNames []string, variant string, caseInsensitive bool, +) (string, error) { required := make(map[string]struct{}, len(archiveNames)) - targets := make(map[string]*zip.File, len(extractNames)) for _, name := range archiveNames { required[name] = struct{}{} } - for _, f := range r.File { - name, err := validateWindowsZipMember(f.Name) - if err != nil { - return err + seen := make(map[string]struct{}, len(names)) + found := make(map[string]struct{}, len(archiveNames)) + uiPack := "" + for _, name := range names { + key := name + if caseInsensitive { + key = strings.ToLower(name) } - key := strings.ToLower(strings.TrimSuffix(name, "/")) - if _, ok := seen[key]; ok { - return fmt.Errorf("duplicate or case-conflicting zip entry: %q", f.Name) + if _, exists := seen[key]; exists { + return "", fmt.Errorf( + "duplicate or case-conflicting archive entry: %q", name, + ) } seen[key] = struct{}{} - if _, ok := required[name]; !ok || f.FileInfo().IsDir() { - return fmt.Errorf( + _, isRequired := required[name] + isPack := variant == "ui" && isUIPackName(name) + if !isRequired && !isPack { + return "", fmt.Errorf( "archive must contain only the exact root files: %s", strings.Join(archiveNames, ", "), ) } - for _, extractName := range extractNames { - if name == extractName { - targets[name] = f + if isPack { + if uiPack != "" { + return "", fmt.Errorf( + "archive must contain exactly one content-addressed UI pack", + ) } + uiPack = name + } else { + found[name] = struct{}{} } } - if len(seen) != len(archiveNames) || len(targets) != len(extractNames) { - return fmt.Errorf( + expectedCount := len(archiveNames) + if variant == "ui" { + expectedCount++ + } + if len(seen) != expectedCount || len(found) != len(required) || + (variant == "ui" && uiPack == "") { + return "", fmt.Errorf( "archive must contain exactly one of each required root file: %s", strings.Join(archiveNames, ", "), ) } - - for _, name := range extractNames { - target := targets[name] - rc, err := target.Open() - if err != nil { - return err - } - outputPath := filepath.Join(destDir, name) - out, err := os.OpenFile( - outputPath, - os.O_WRONLY|os.O_CREATE|os.O_EXCL, - 0600, - ) - if err != nil { - rc.Close() - return err - } - _, copyErr := io.Copy(out, rc) //nolint:gosec - closeOutErr := out.Close() - closeReadErr := rc.Close() - if copyErr != nil { - os.Remove(outputPath) - return copyErr - } - if closeOutErr != nil { - os.Remove(outputPath) - return closeOutErr - } - if closeReadErr != nil { - os.Remove(outputPath) - return closeReadErr - } - } - return nil + return uiPack, nil } -func validateWindowsZipMember(raw string) (string, error) { - name := strings.ReplaceAll(raw, "\\", "/") - segmentsName := strings.TrimSuffix(name, "/") - if segmentsName == "" || strings.HasPrefix(name, "/") || strings.Contains(name, ":") { - return "", fmt.Errorf("unsafe zip entry path: %q", raw) - } - for _, segment := range strings.Split(segmentsName, "/") { - if segment == "" || segment == "." || segment == ".." || - strings.HasSuffix(segment, ".") || strings.HasSuffix(segment, " ") { - return "", fmt.Errorf("unsafe zip entry path: %q", raw) - } +func runtimeArchiveNames(extractNames []string, uiPack string) []string { + result := append([]string(nil), extractNames...) + if uiPack != "" { + result = append(result, uiPack) } - return name, nil + return result } -func verifyCandidate(path string) error { - ctx, cancel := context.WithTimeout(context.Background(), candidateTimeout) - defer cancel() - cmd := exec.CommandContext(ctx, path, "--version") - cmd.Stdin = nil - cmd.Stdout = io.Discard - cmd.Stderr = io.Discard - if err := cmd.Run(); err != nil { - if ctx.Err() != nil { - return fmt.Errorf("downloaded binary verification timed out: %w", ctx.Err()) - } - return fmt.Errorf("downloaded binary failed to run: %w", err) +func validateArchiveResourceLimits(limits archiveResourceLimits) error { + if limits.compressedBytes <= 0 || limits.members <= 0 || + limits.memberBytes <= 0 || limits.expandedBytes <= 0 { + return fmt.Errorf("invalid archive resource safety limits") } return nil } -func fileSHA256(path string) ([sha256.Size]byte, error) { - var digest [sha256.Size]byte - source, err := os.Open(path) +func requireCompressedArchiveWithinLimit( + archivePath string, limits archiveResourceLimits, +) error { + status, err := os.Stat(archivePath) if err != nil { - return digest, err + return err } - defer source.Close() - hash := sha256.New() - if _, err := io.Copy(hash, source); err != nil { - return digest, err + if !status.Mode().IsRegular() { + return fmt.Errorf("release archive is not a regular file") } - copy(digest[:], hash.Sum(nil)) - return digest, nil + if status.Size() > limits.compressedBytes { + return fmt.Errorf( + "release archive exceeds the %d-byte compressed safety limit", + limits.compressedBytes, + ) + } + return nil } -func filesEqualSHA256(left, right string) (bool, error) { - leftInfo, err := os.Stat(left) - if err != nil { - return false, err - } - rightInfo, err := os.Stat(right) - if err != nil { - return false, err - } - if !leftInfo.Mode().IsRegular() || !rightInfo.Mode().IsRegular() || - leftInfo.Size() != rightInfo.Size() { - return false, nil +func accountDeclaredArchiveMember( + name string, size int64, expanded *int64, limits archiveResourceLimits, +) error { + if size < 0 { + return fmt.Errorf("archive member has a negative declared size: %q", name) } - leftDigest, err := fileSHA256(left) - if err != nil { - return false, err + if size > limits.memberBytes { + return fmt.Errorf( + "archive member %q exceeds the %d-byte expanded safety limit", + name, limits.memberBytes, + ) } - rightDigest, err := fileSHA256(right) - if err != nil { - return false, err + if *expanded > limits.expandedBytes-size { + return fmt.Errorf( + "archive exceeds the %d-byte aggregate expanded safety limit", + limits.expandedBytes, + ) } - return leftDigest == rightDigest, nil + *expanded += size + return nil } -func installCandidateAtomically(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - pattern := "." + filepath.Base(dst) + ".*.tmp" - if runtime.GOOS == "windows" { - pattern += ".exe" +func copyArchiveMemberWithLimits( + destination io.Writer, + source io.Reader, + name string, + declaredSize int64, + actualExpanded *int64, + limits archiveResourceLimits, +) error { + remaining := limits.expandedBytes - *actualExpanded + copyLimit := limits.memberBytes + if remaining < copyLimit { + copyLimit = remaining } - out, err := os.CreateTemp(filepath.Dir(dst), pattern) + copied, err := io.Copy( + destination, io.LimitReader(source, copyLimit+1), + ) if err != nil { return err } - staged := out.Name() - defer os.Remove(staged) - if _, err := io.Copy(out, in); err != nil { - out.Close() - return err - } - if err := out.Chmod(0755); err != nil { - out.Close() - return err - } - if err := out.Sync(); err != nil { - out.Close() - return err - } - if err := out.Close(); err != nil { - return err + if copied > limits.memberBytes { + return fmt.Errorf( + "archive member %q exceeds the %d-byte actual expanded safety limit", + name, limits.memberBytes, + ) } - if err := verifyCandidate(staged); err != nil { - return err + if copied > remaining { + return fmt.Errorf( + "archive exceeds the %d-byte aggregate actual expanded safety limit", + limits.expandedBytes, + ) } - if err := os.Rename(staged, dst); err != nil { - identical, compareErr := filesEqualSHA256(staged, dst) - if compareErr == nil && identical { - return verifyCandidate(dst) - } - return err + if copied != declaredSize { + return fmt.Errorf( + "archive member %q actual size %d does not match declared size %d", + name, copied, declaredSize, + ) } + *actualExpanded += copied return nil } -func windowsPairToken() (string, error) { - raw := make([]byte, windowsPairTransactionTagSize) - if _, err := rand.Read(raw); err != nil { - return "", err - } - return hex.EncodeToString(raw), nil +func tarGzMemberNames(archivePath string) ([]string, error) { + return tarGzMemberNamesWithLimits( + archivePath, defaultArchiveResourceLimits, + ) } -func windowsPairLockOwner(lockPath string) (int, string, bool) { - data, err := os.ReadFile(filepath.Join(lockPath, windowsPairLockOwnerFile)) - if err != nil { - return 0, "", false +func tarGzMemberNamesWithLimits( + archivePath string, limits archiveResourceLimits, +) ([]string, error) { + if err := validateArchiveResourceLimits(limits); err != nil { + return nil, err } - var pid int - var token string - var extra string - count, _ := fmt.Sscanf(string(data), "%d %s %s", &pid, &token, &extra) - decoded, decodeErr := hex.DecodeString(token) - if count != 2 || pid <= 0 || decodeErr != nil || - len(decoded) != windowsPairTransactionTagSize { - return 0, "", false + if err := requireCompressedArchiveWithinLimit(archivePath, limits); err != nil { + return nil, err } - return pid, token, true -} - -func windowsPairTryReclaimLock(lockPath, contenderToken string) bool { - pid, _, ownerOK := windowsPairLockOwner(lockPath) - reclaim := ownerOK && !cbmWindowsLockProcessAlive(pid) - if !ownerOK { - status, err := os.Stat(lockPath) - if os.IsNotExist(err) { - return true - } - if err != nil { - return false + f, err := os.Open(archivePath) + if err != nil { + return nil, err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gz.Close() + tr := tar.NewReader(gz) + var names []string + var declaredExpanded int64 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + return nil, fmt.Errorf( + "archive member is not a regular root file: %q", hdr.Name, + ) + } + if len(names) >= limits.members { + return nil, fmt.Errorf( + "archive exceeds the %d-member safety limit", limits.members, + ) + } + if err := accountDeclaredArchiveMember( + hdr.Name, hdr.Size, &declaredExpanded, limits, + ); err != nil { + return nil, err + } + names = append(names, hdr.Name) + } + return names, nil +} + +func extractTarGz( + archivePath, destDir string, + archiveNames, extractNames []string, + variant string, +) ([]string, error) { + return extractTarGzWithLimits( + archivePath, + destDir, + archiveNames, + extractNames, + variant, + defaultArchiveResourceLimits, + ) +} + +func extractTarGzWithLimits( + archivePath, destDir string, + archiveNames, extractNames []string, + variant string, + limits archiveResourceLimits, +) ([]string, error) { + names, err := tarGzMemberNamesWithLimits(archivePath, limits) + if err != nil { + return nil, err + } + uiPack, err := validateArchiveMemberNames( + names, archiveNames, variant, false, + ) + if err != nil { + return nil, err + } + runtimeNames := runtimeArchiveNames(extractNames, uiPack) + targets := make(map[string]struct{}, len(runtimeNames)) + for _, name := range runtimeNames { + targets[name] = struct{}{} + } + + f, err := os.Open(archivePath) + if err != nil { + return nil, err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gz.Close() + tr := tar.NewReader(gz) + extracted := make(map[string]struct{}, len(runtimeNames)) + memberCount := 0 + var declaredExpanded int64 + var actualExpanded int64 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + memberCount++ + if memberCount > limits.members { + return nil, fmt.Errorf( + "archive exceeds the %d-member safety limit", limits.members, + ) + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + return nil, fmt.Errorf( + "archive member is not a regular root file: %q", hdr.Name, + ) + } + if err := accountDeclaredArchiveMember( + hdr.Name, hdr.Size, &declaredExpanded, limits, + ); err != nil { + return nil, err + } + if _, wanted := targets[hdr.Name]; wanted { + outputPath := filepath.Join(destDir, hdr.Name) + out, err := os.OpenFile( + outputPath, + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0600, + ) + if err != nil { + return nil, err + } + copyErr := copyArchiveMemberWithLimits( + out, tr, hdr.Name, hdr.Size, &actualExpanded, limits, + ) + closeErr := out.Close() + if copyErr != nil { + _ = os.Remove(outputPath) + return nil, copyErr + } + if closeErr != nil { + _ = os.Remove(outputPath) + return nil, closeErr + } + extracted[hdr.Name] = struct{}{} + } else if err := copyArchiveMemberWithLimits( + io.Discard, tr, hdr.Name, hdr.Size, &actualExpanded, limits, + ); err != nil { + return nil, err + } + } + if len(extracted) != len(runtimeNames) { + return nil, fmt.Errorf("archive runtime set extraction was incomplete") + } + return runtimeNames, nil +} + +func extractZip( + archivePath, destDir string, + archiveNames, extractNames []string, + variant string, +) ([]string, error) { + return extractZipWithLimits( + archivePath, + destDir, + archiveNames, + extractNames, + variant, + defaultArchiveResourceLimits, + ) +} + +func extractZipWithLimits( + archivePath, destDir string, + archiveNames, extractNames []string, + variant string, + limits archiveResourceLimits, +) ([]string, error) { + if err := validateArchiveResourceLimits(limits); err != nil { + return nil, err + } + if err := requireCompressedArchiveWithinLimit(archivePath, limits); err != nil { + return nil, err + } + r, err := zip.OpenReader(archivePath) + if err != nil { + return nil, err + } + defer r.Close() + if len(r.File) > limits.members { + return nil, fmt.Errorf( + "archive exceeds the %d-member safety limit", limits.members, + ) + } + names := make([]string, 0, len(r.File)) + declaredSizes := make(map[string]int64, len(r.File)) + var declaredExpanded int64 + for _, f := range r.File { + name, err := validateWindowsZipMember(f.Name) + if err != nil { + return nil, err + } + if !f.FileInfo().Mode().IsRegular() { + return nil, fmt.Errorf( + "archive member is not a regular root file: %q", f.Name, + ) + } + if f.UncompressedSize64 > uint64(limits.memberBytes) { + return nil, fmt.Errorf( + "archive member %q exceeds the %d-byte expanded safety limit", + name, limits.memberBytes, + ) + } + declaredSize := int64(f.UncompressedSize64) + if err := accountDeclaredArchiveMember( + name, declaredSize, &declaredExpanded, limits, + ); err != nil { + return nil, err + } + names = append(names, name) + declaredSizes[name] = declaredSize + } + uiPack, err := validateArchiveMemberNames( + names, archiveNames, variant, true, + ) + if err != nil { + return nil, err + } + runtimeNames := runtimeArchiveNames(extractNames, uiPack) + targetNames := make(map[string]struct{}, len(runtimeNames)) + for _, name := range runtimeNames { + targetNames[name] = struct{}{} + } + extracted := make(map[string]struct{}, len(runtimeNames)) + var actualExpanded int64 + + for index, target := range r.File { + name := names[index] + rc, err := target.Open() + if err != nil { + return nil, err + } + var destination io.Writer = io.Discard + var out *os.File + outputPath := "" + if _, wanted := targetNames[name]; wanted { + outputPath = filepath.Join(destDir, name) + out, err = os.OpenFile( + outputPath, + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0600, + ) + if err != nil { + _ = rc.Close() + return nil, err + } + destination = out + } + copyErr := copyArchiveMemberWithLimits( + destination, + rc, + name, + declaredSizes[name], + &actualExpanded, + limits, + ) + var closeOutErr error + if out != nil { + closeOutErr = out.Close() + } + closeReadErr := rc.Close() + if copyErr != nil { + if outputPath != "" { + _ = os.Remove(outputPath) + } + return nil, copyErr + } + if closeOutErr != nil { + _ = os.Remove(outputPath) + return nil, closeOutErr + } + if closeReadErr != nil { + if outputPath != "" { + _ = os.Remove(outputPath) + } + return nil, closeReadErr + } + if outputPath != "" { + extracted[name] = struct{}{} + } + } + if len(extracted) != len(runtimeNames) { + return nil, fmt.Errorf("archive runtime set extraction was incomplete") + } + return runtimeNames, nil +} + +func validateWindowsZipMember(raw string) (string, error) { + name := strings.ReplaceAll(raw, "\\", "/") + segmentsName := strings.TrimSuffix(name, "/") + if segmentsName == "" || strings.HasPrefix(name, "/") || strings.Contains(name, ":") { + return "", fmt.Errorf("unsafe zip entry path: %q", raw) + } + for _, segment := range strings.Split(segmentsName, "/") { + if segment == "" || segment == "." || segment == ".." || + strings.HasSuffix(segment, ".") || strings.HasSuffix(segment, " ") { + return "", fmt.Errorf("unsafe zip entry path: %q", raw) + } + } + return name, nil +} + +func verifyCandidate(path string) error { + ctx, cancel := context.WithTimeout(context.Background(), candidateTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, path, "--verify-runtime-assets") + cmd.Stdin = nil + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return fmt.Errorf("downloaded binary verification timed out: %w", ctx.Err()) + } + return fmt.Errorf("downloaded binary failed to run: %w", err) + } + return nil +} + +func fileSHA256(path string) ([sha256.Size]byte, error) { + var digest [sha256.Size]byte + source, err := os.Open(path) + if err != nil { + return digest, err + } + defer source.Close() + hash := sha256.New() + if _, err := io.Copy(hash, source); err != nil { + return digest, err + } + copy(digest[:], hash.Sum(nil)) + return digest, nil +} + +func filesEqualSHA256(left, right string) (bool, error) { + leftInfo, err := os.Lstat(left) + if err != nil { + return false, err + } + rightInfo, err := os.Lstat(right) + if err != nil { + return false, err + } + if !leftInfo.Mode().IsRegular() || !rightInfo.Mode().IsRegular() || + !platformRuntimeSetFileLinkCountOne(left, leftInfo) || + !platformRuntimeSetFileLinkCountOne(right, rightInfo) || + leftInfo.Size() != rightInfo.Size() { + return false, nil + } + leftDigest, err := fileSHA256(left) + if err != nil { + return false, err + } + rightDigest, err := fileSHA256(right) + if err != nil { + return false, err + } + return leftDigest == rightDigest, nil +} + +func regularRuntimeFile(path string) bool { + status, err := os.Lstat(path) + return err == nil && status.Mode().IsRegular() && + platformRuntimeSetFileLinkCountOne(path, status) +} + +func uiPackMatchesDigest(directory, name string) bool { + digest, err := fileSHA256(filepath.Join(directory, name)) + if err != nil { + return false + } + expected := strings.TrimSuffix(strings.TrimPrefix(name, "cbm-ui-"), ".pack") + return hex.EncodeToString(digest[:]) == expected +} + +func runtimeSetNames( + directory, binaryName, variant string, +) ([]string, bool) { + if !regularRuntimeFile(filepath.Join(directory, binaryName)) || + !regularRuntimeFile(filepath.Join(directory, integrationsFileName)) { + return nil, false + } + entries, err := os.ReadDir(directory) + if err != nil { + return nil, false + } + var packLike []string + var validPacks []string + for _, entry := range entries { + name := entry.Name() + if strings.HasPrefix(name, "cbm-ui-") && strings.HasSuffix(name, ".pack") { + packLike = append(packLike, name) + if isUIPackName(name) && + regularRuntimeFile(filepath.Join(directory, name)) && + uiPackMatchesDigest(directory, name) { + validPacks = append(validPacks, name) + } + } + } + if variant == "ui" { + if len(packLike) != 1 || len(validPacks) != 1 { + return nil, false + } + } else if len(packLike) != 0 { + return nil, false + } + names := []string{integrationsFileName} + if variant == "ui" { + names = append(names, validPacks[0]) + } + names = append(names, binaryName) + return names, true +} + +func runtimeSetReady( + directory, binaryName, variant string, verifier func(string) error, +) bool { + if _, ok := runtimeSetNames(directory, binaryName, variant); !ok { + return false + } + return verifier == nil || verifier(filepath.Join(directory, binaryName)) == nil +} + +func requireSafeRuntimeDirectory(directory string) error { + status, err := os.Lstat(directory) + if err != nil { + return err + } + if !status.IsDir() || status.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing unsafe package-cache directory: %s", directory) + } + return nil +} + +func runtimeSetReadyLocked( + directory, binaryName, variant string, verifier func(string) error, +) (ready bool, result error) { + if err := os.MkdirAll(directory, 0755); err != nil { + return false, err + } + if err := requireSafeRuntimeDirectory(directory); err != nil { + return false, err + } + lock, err := acquireRuntimeSetLock(directory) + if err != nil { + return false, err + } + defer func() { + attachRuntimeLockReleaseError(&result, releaseRuntimeSetLock(lock)) + }() + if err := refreshRuntimeSetLock(lock); err != nil { + return false, err + } + if err := reconcileRuntimeBackups( + directory, binaryName, variant, verifier, lock, + ); err != nil { + return false, err + } + return runtimeSetReady(directory, binaryName, variant, verifier), nil +} + +func copyRuntimeStage( + sourcePath, destinationDirectory string, executable bool, + maintainLease func() error, +) (string, error) { + input, err := os.Open(sourcePath) + if err != nil { + return "", err + } + defer input.Close() + pattern := ".cbm-runtime-stage-*" + if executable && runtime.GOOS == "windows" { + pattern += ".exe" + } + output, err := os.CreateTemp(destinationDirectory, pattern) + if err != nil { + return "", err + } + stagedPath := output.Name() + mode := os.FileMode(0644) + if executable { + mode = 0755 + } + buffer := make([]byte, 1024*1024) + lastRefresh := time.Now() + var copyErr error + for { + count, readErr := input.Read(buffer) + if count > 0 { + written := 0 + for written < count { + amount, writeErr := output.Write(buffer[written:count]) + if writeErr != nil { + copyErr = writeErr + break + } + written += amount + } + } + if copyErr != nil || readErr == io.EOF { + break + } + if readErr != nil { + copyErr = readErr + break + } + if maintainLease != nil && time.Since(lastRefresh) >= runtimeSetLockHeartbeat { + if err := maintainLease(); err != nil { + copyErr = err + break + } + lastRefresh = time.Now() + } + } + chmodErr := output.Chmod(mode) + syncErr := output.Sync() + closeErr := output.Close() + for _, candidate := range []error{copyErr, chmodErr, syncErr, closeErr} { + if candidate != nil { + _ = os.Remove(stagedPath) + return "", candidate + } + } + return stagedPath, nil +} + +// A grouped backup is a small process-crash journal. runtimeBackupRetired +// separates a partial retirement from partial publication; +// runtimeBackupCleanupOnly makes deletion retryable after the destination has +// been accepted or restored. +func runtimeBackupDirectoryName(name string) bool { + if !strings.HasPrefix(name, runtimeBackupPrefix) { + return false + } + token := strings.TrimPrefix(name, runtimeBackupPrefix) + decoded, err := hex.DecodeString(token) + return err == nil && token == strings.ToLower(token) && + len(decoded) == runtimeSetLockTokenSize +} + +func runtimeBackupTargetName(name, binaryName string) bool { + return name == binaryName || name == integrationsFileName || + (strings.HasPrefix(name, "cbm-ui-") && strings.HasSuffix(name, ".pack")) +} + +func createRuntimeBackupDirectory(directory string) (string, error) { + for attempt := 0; attempt < 128; attempt++ { + token, err := runtimeSetLockToken() + if err != nil { + return "", err + } + backupDirectory := filepath.Join(directory, runtimeBackupPrefix+token) + if err := os.Mkdir(backupDirectory, 0700); err == nil { + return backupDirectory, nil + } else if !os.IsExist(err) { + return "", err + } + } + return "", fmt.Errorf("could not reserve a package-cache backup transaction") +} + +func createRuntimeBackupMarker(backupDirectory, name string) error { + marker, err := os.OpenFile( + filepath.Join(backupDirectory, name), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0600, + ) + if err != nil { + return err + } + syncErr := marker.Sync() + closeErr := marker.Close() + if syncErr != nil { + return syncErr + } + return closeErr +} + +func inspectRuntimeBackupDirectory( + directory, entryName, binaryName string, +) (*runtimeBackupTransaction, error) { + backupDirectory := filepath.Join(directory, entryName) + directoryStatus, err := os.Lstat(backupDirectory) + if err != nil || !directoryStatus.IsDir() || + directoryStatus.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf( + "refusing unsafe package-cache backup: %s", backupDirectory, + ) + } + backup := &runtimeBackupTransaction{ + path: backupDirectory, files: make(map[string]runtimeBackupFile), + } + entries, err := os.ReadDir(backupDirectory) + if err != nil { + return nil, err + } + for _, entry := range entries { + name := entry.Name() + candidate := filepath.Join(backupDirectory, name) + status, err := os.Lstat(candidate) + if err != nil { + return nil, err + } + if name == runtimeBackupRetired || name == runtimeBackupCleanupOnly { + if !status.Mode().IsRegular() || + !platformRuntimeSetFileLinkCountOne(candidate, status) || + status.Size() != 0 { + return nil, fmt.Errorf( + "refusing unsafe package-cache backup marker: %s", candidate, + ) + } + if name == runtimeBackupRetired { + backup.retired = true + } else { + backup.cleanupOnly = true + } + continue + } + if !runtimeBackupTargetName(name, binaryName) || + !status.Mode().IsRegular() || + !platformRuntimeSetFileLinkCountOne(candidate, status) { + return nil, fmt.Errorf( + "refusing unsafe package-cache backup member: %s", candidate, + ) + } + digest, err := fileSHA256(candidate) + if err != nil { + return nil, err + } + backup.files[name] = runtimeBackupFile{path: candidate, digest: digest} + } + return backup, nil +} + +func listRuntimeBackupDirectories( + directory, binaryName string, +) ([]*runtimeBackupTransaction, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + var names []string + for _, entry := range entries { + if runtimeBackupDirectoryName(entry.Name()) { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + backups := make([]*runtimeBackupTransaction, 0, len(names)) + for _, name := range names { + backup, err := inspectRuntimeBackupDirectory(directory, name, binaryName) + if err != nil { + return nil, err + } + backups = append(backups, backup) + } + return backups, nil +} + +func currentRuntimeTargets( + directory, binaryName string, +) (map[string]string, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + targets := make(map[string]string) + for _, entry := range entries { + name := entry.Name() + if !runtimeBackupTargetName(name, binaryName) { + continue + } + target := filepath.Join(directory, name) + status, err := os.Lstat(target) + if err != nil || !status.Mode().IsRegular() || + !platformRuntimeSetFileLinkCountOne(target, status) { + return nil, fmt.Errorf("refusing unsafe package-cache target: %s", target) + } + targets[name] = target + } + return targets, nil +} + +func cleanupRuntimeBackup( + backup *runtimeBackupTransaction, lock *runtimeSetLock, +) error { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if !backup.cleanupOnly { + if err := createRuntimeBackupMarker( + backup.path, runtimeBackupCleanupOnly, + ); err != nil { + return err + } + backup.cleanupOnly = true + } + names := make([]string, 0, len(backup.files)) + for name := range backup.files { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + member := backup.files[name] + if !pathMatchesSHA256(member.path, member.digest) { + return fmt.Errorf( + "package-cache backup member changed during cleanup: %s", name, + ) + } + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := os.Remove(member.path); err != nil { + return err + } + if runtimeBackupCrashObserver != nil { + if err := runtimeBackupCrashObserver( + runtimeBackupCleanupRemovedEvent, member.path, + ); err != nil { + return err + } + } + } + if backup.retired { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := os.Remove(filepath.Join(backup.path, runtimeBackupRetired)); err != nil { + return err + } + } + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := os.Remove(filepath.Join(backup.path, runtimeBackupCleanupOnly)); err != nil { + return err + } + return os.Remove(backup.path) +} + +func copyRuntimeBackupFile( + member runtimeBackupFile, target string, executable bool, lock *runtimeSetLock, +) error { + staged, err := copyRuntimeStage( + member.path, + filepath.Dir(target), + executable, + func() error { return refreshRuntimeSetLock(lock) }, + ) + if err != nil { + return err + } + defer os.Remove(staged) + stagedDigest, err := fileSHA256(staged) + if err != nil { + return err + } + if stagedDigest != member.digest { + return fmt.Errorf("package-cache backup changed while it was being restored") + } + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + return os.Rename(staged, target) +} + +func reconcileRuntimeBackups( + directory, binaryName, variant string, + verifier func(string) error, + lock *runtimeSetLock, +) error { + if err := refreshRuntimeSetLock(lock); err != nil { + return err + } + backups, err := listRuntimeBackupDirectories(directory, binaryName) + if err != nil { + return err + } + remaining := make([]*runtimeBackupTransaction, 0, len(backups)) + for _, backup := range backups { + if backup.cleanupOnly { + if err := cleanupRuntimeBackup(backup, lock); err != nil { + return err + } + } else { + remaining = append(remaining, backup) + } + } + backups = remaining + if len(backups) == 0 { + return nil + } + if runtimeSetReady(directory, binaryName, variant, verifier) { + for _, backup := range backups { + if err := cleanupRuntimeBackup(backup, lock); err != nil { + return err + } + } + return nil + } + if len(backups) == 0 { + return nil + } + remaining = remaining[:0] + for _, backup := range backups { + if !backup.retired && len(backup.files) == 0 { + if err := cleanupRuntimeBackup(backup, lock); err != nil { + return err + } + } else { + remaining = append(remaining, backup) + } + } + backups = remaining + if len(backups) == 0 { + return nil + } + if len(backups) != 1 { + return fmt.Errorf( + "multiple package-cache backup transactions require manual recovery", + ) + } + backup := backups[0] + targets, err := currentRuntimeTargets(directory, binaryName) + if err != nil { + return err + } + if !backup.retired { + for name, member := range backup.files { + if target, exists := targets[name]; exists && + !pathMatchesSHA256(target, member.digest) { + return fmt.Errorf( + "package-cache backup conflicts with current target: %s", name, + ) + } + } + } else { + if target, exists := targets[binaryName]; exists { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := os.Remove(target); err != nil { + return err + } + delete(targets, binaryName) + } + names := make([]string, 0, len(targets)) + for name := range targets { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := os.Remove(targets[name]); err != nil { + return err + } + } + } + restoreNames := make([]string, 0, len(backup.files)) + for name := range backup.files { + if name != binaryName { + restoreNames = append(restoreNames, name) + } + } + sort.Strings(restoreNames) + if _, exists := backup.files[binaryName]; exists { + restoreNames = append(restoreNames, binaryName) + } + for _, name := range restoreNames { + member := backup.files[name] + target := filepath.Join(directory, name) + if !pathMatchesSHA256(target, member.digest) { + if err := copyRuntimeBackupFile( + member, target, name == binaryName, lock, + ); err != nil { + return err + } + } + } + for name, member := range backup.files { + if !pathMatchesSHA256(filepath.Join(directory, name), member.digest) { + return fmt.Errorf("package-cache backup restoration failed: %s", name) + } + } + if err := cleanupRuntimeBackup(backup, lock); err != nil { + return err + } + return nil +} + +func pathMatchesSHA256(path string, expected [sha256.Size]byte) bool { + if !regularRuntimeFile(path) { + return false + } + actual, err := fileSHA256(path) + return err == nil && actual == expected +} + +func runtimeSetLockToken() (string, error) { + raw := make([]byte, runtimeSetLockTokenSize) + if _, err := rand.Read(raw); err != nil { + return "", err + } + return hex.EncodeToString(raw), nil +} + +func runtimeSetLockOwner(lockPath string) (runtimeSetLockOwnerRecord, bool) { + var empty runtimeSetLockOwnerRecord + status, err := os.Lstat(lockPath) + if err != nil || status.Mode()&os.ModeSymlink != 0 || + (!status.Mode().IsRegular() && !status.IsDir()) { + return empty, false + } + ownerPath := lockPath + if status.IsDir() { + ownerPath = filepath.Join(lockPath, runtimeSetLockOwnerFile) + } + ownerStatus, err := os.Lstat(ownerPath) + if err != nil || !ownerStatus.Mode().IsRegular() { + return empty, false + } + data, err := os.ReadFile(ownerPath) + if err != nil { + return empty, false + } + var record runtimeSetLockOwnerRecord + if err := json.Unmarshal(data, &record); err != nil { + return empty, false + } + decoded, decodeErr := hex.DecodeString(record.Token) + if record.PID <= 0 || record.Token != strings.ToLower(record.Token) || + decodeErr != nil || len(decoded) != runtimeSetLockTokenSize { + return empty, false + } + return record, true +} + +func runtimeSetSameLockObject(left, right os.FileInfo) bool { + return left != nil && right != nil && os.SameFile(left, right) && + left.Mode().Type() == right.Mode().Type() +} + +func writeRuntimeSetLockRecord(owner *os.File, token string) error { + if err := owner.Truncate(0); err != nil { + return err + } + if _, err := owner.Seek(0, io.SeekStart); err != nil { + return err + } + if err := json.NewEncoder(owner).Encode(runtimeSetLockOwnerRecord{ + PID: os.Getpid(), + Token: token, + LeaseExpires: time.Now().Add(runtimeSetLockLease).UnixMilli(), + }); err != nil { + return err + } + return owner.Sync() +} + +// writeRuntimeSetLockOwner creates the legacy directory-owner shape used by +// migration tests and previous wrapper releases. +func writeRuntimeSetLockOwner(lockPath, token string) error { + owner, err := os.OpenFile( + filepath.Join(lockPath, runtimeSetLockOwnerFile), + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + 0600, + ) + if err != nil { + return err + } + writeErr := json.NewEncoder(owner).Encode(runtimeSetLockOwnerRecord{ + PID: os.Getpid(), Token: token, + }) + syncErr := owner.Sync() + closeErr := owner.Close() + for _, candidate := range []error{writeErr, syncErr, closeErr} { + if candidate != nil { + return candidate } - reclaim = time.Since(status.ModTime()) >= windowsPairOwnerlessStale } - if !reclaim { + return nil +} + +func runtimeSetOwnerReclaimable( + status os.FileInfo, owner runtimeSetLockOwnerRecord, ownerOK bool, +) bool { + if ownerOK { + // Lease expiry and age are not evidence that a syntactically valid + // owner is dead. A paused live publisher can resume between lock-owner + // assertions, so reclaim only when the platform proves its PID is gone. + return !runtimeSetProcessAlive(owner.PID) + } + return time.Since(status.ModTime()) >= runtimeSetOwnerlessStale +} + +func restoreDisplacedRuntimeSetLock(reclaimed, lockPath string, status os.FileInfo) { + if status == nil || !status.Mode().IsRegular() { + return + } + if err := os.Link(reclaimed, lockPath); err == nil { + _ = os.Remove(reclaimed) + } +} + +func runtimeSetTryReclaimLock(lockPath, contenderToken string) bool { + observedStatus, err := os.Lstat(lockPath) + if os.IsNotExist(err) { + return true + } + if err != nil || observedStatus.Mode()&os.ModeSymlink != 0 || + (!observedStatus.Mode().IsRegular() && !observedStatus.IsDir()) { + return false + } + owner, ownerOK := runtimeSetLockOwner(lockPath) + if !runtimeSetOwnerReclaimable(observedStatus, owner, ownerOK) { return false } reclaimed := lockPath + ".reclaimed-" + contenderToken if err := os.Rename(lockPath, reclaimed); err != nil { return os.IsNotExist(err) } - _ = os.RemoveAll(reclaimed) + movedStatus, movedErr := os.Lstat(reclaimed) + if movedErr != nil || !runtimeSetSameLockObject(observedStatus, movedStatus) { + restoreDisplacedRuntimeSetLock(reclaimed, lockPath, movedStatus) + return false + } + movedOwner, movedOwnerOK := runtimeSetLockOwner(reclaimed) + if !runtimeSetOwnerReclaimable(movedStatus, movedOwner, movedOwnerOK) { + restoreDisplacedRuntimeSetLock(reclaimed, lockPath, movedStatus) + return false + } + if movedStatus.IsDir() { + _ = os.RemoveAll(reclaimed) + } else { + _ = os.Remove(reclaimed) + } return true } -func acquireWindowsPairLock(dstDir string) (*windowsPairLock, error) { - token, err := windowsPairToken() +func assertRuntimeSetLockOwner(lock *runtimeSetLock) error { + if lock == nil || lock.file == nil { + return fmt.Errorf("missing package-cache runtime-set publication lock") + } + descriptorStatus, descriptorErr := lock.file.Stat() + canonicalStatus, canonicalErr := os.Lstat(lock.path) + owner, ownerOK := runtimeSetLockOwner(lock.path) + if descriptorErr != nil || canonicalErr != nil || + !runtimeSetSameLockObject(descriptorStatus, canonicalStatus) || + !ownerOK || owner.PID != os.Getpid() || owner.Token != lock.token { + return fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) + } + return nil +} + +func refreshRuntimeSetLock(lock *runtimeSetLock) error { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if err := writeRuntimeSetLockRecord(lock.file, lock.token); err != nil { + return err + } + return assertRuntimeSetLockOwner(lock) +} + +func acquireRuntimeSetLock(destinationDirectory string) (*runtimeSetLock, error) { + token, err := runtimeSetLockToken() if err != nil { return nil, err } - lockPath := filepath.Join(dstDir, windowsPairLockName) - deadline := time.Now().Add(windowsPairLockWait) + lockPath := filepath.Join(destinationDirectory, runtimeSetLockName) + claimPath := lockPath + ".claim-" + token + deadline := time.Now().Add(runtimeSetLockWait) for { - if err := os.Mkdir(lockPath, 0700); err == nil { - owner := []byte(fmt.Sprintf("%d %s\n", os.Getpid(), token)) - ownerPath := filepath.Join(lockPath, windowsPairLockOwnerFile) - if err := os.WriteFile(ownerPath, owner, 0600); err != nil { - _ = os.RemoveAll(lockPath) + owner, claimErr := os.OpenFile( + claimPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600, + ) + if claimErr != nil { + return nil, claimErr + } + if err := writeRuntimeSetLockRecord(owner, token); err != nil { + _ = owner.Close() + _ = os.Remove(claimPath) + return nil, err + } + if runtimeSetLockClaimObserver != nil { + if err := runtimeSetLockClaimObserver(); err != nil { + _ = owner.Close() + _ = os.Remove(claimPath) return nil, err } - return &windowsPairLock{path: lockPath, token: token}, nil - } else if !os.IsExist(err) { - return nil, err } - if windowsPairTryReclaimLock(lockPath, token) { + linkErr := os.Link(claimPath, lockPath) + if linkErr == nil { + if err := os.Remove(claimPath); err != nil { + _ = owner.Close() + return nil, err + } + descriptorStatus, descriptorErr := owner.Stat() + canonicalStatus, canonicalErr := os.Lstat(lockPath) + if descriptorErr != nil || canonicalErr != nil || + !runtimeSetSameLockObject(descriptorStatus, canonicalStatus) { + _ = owner.Close() + return nil, fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) + } + return &runtimeSetLock{path: lockPath, token: token, file: owner}, nil + } + _ = owner.Close() + _ = os.Remove(claimPath) + if !os.IsExist(linkErr) { + return nil, linkErr + } + if runtimeSetTryReclaimLock(lockPath, token) { continue } + if runtimeSetLockWaitObserver != nil { + runtimeSetLockWaitObserver() + } if !time.Now().Before(deadline) { return nil, fmt.Errorf( - "timed out waiting for Windows package-cache publication lock") + "timed out waiting for package-cache runtime-set publication lock", + ) } - time.Sleep(windowsPairLockPoll) + time.Sleep(runtimeSetLockPoll) } } -func releaseWindowsPairLock(lock *windowsPairLock) error { - if lock == nil { - return fmt.Errorf("missing Windows package-cache publication lock") - } - pid, token, ok := windowsPairLockOwner(lock.path) - if !ok || pid != os.Getpid() || token != lock.token { - return fmt.Errorf("Windows package-cache publication lock ownership changed") - } - if err := os.Remove(filepath.Join(lock.path, windowsPairLockOwnerFile)); err != nil { +func releaseRuntimeSetLock(lock *runtimeSetLock) error { + if err := assertRuntimeSetLockOwner(lock); err != nil { + if lock != nil && lock.file != nil { + _ = lock.file.Close() + } return err } - return os.Remove(lock.path) -} - -func windowsPairReady(dstDir string, verifier func(string) error) bool { - launcher := filepath.Join(dstDir, windowsLauncherName) - payload := filepath.Join(dstDir, windowsPayloadName) - launcherStatus, launcherErr := os.Lstat(launcher) - payloadStatus, payloadErr := os.Lstat(payload) - return launcherErr == nil && payloadErr == nil && - launcherStatus.Mode().IsRegular() && payloadStatus.Mode().IsRegular() && - verifier(launcher) == nil -} - -func copyWindowsPairStage(src, dst string) error { - input, err := os.Open(src) - if err != nil { + released := lock.path + ".released-" + lock.token + if err := os.Rename(lock.path, released); err != nil { + _ = lock.file.Close() return err } - output, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - _ = input.Close() - return err + descriptorStatus, descriptorErr := lock.file.Stat() + releasedStatus, releasedErr := os.Lstat(released) + owner, ownerOK := runtimeSetLockOwner(released) + if descriptorErr != nil || releasedErr != nil || + !runtimeSetSameLockObject(descriptorStatus, releasedStatus) || + !ownerOK || owner.PID != os.Getpid() || owner.Token != lock.token { + restoreDisplacedRuntimeSetLock(released, lock.path, releasedStatus) + _ = lock.file.Close() + return fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) } - _, copyErr := io.Copy(output, input) - closeInputErr := input.Close() - chmodErr := output.Chmod(0755) - syncErr := output.Sync() - closeOutputErr := output.Close() - for _, candidate := range []error{ - copyErr, closeInputErr, chmodErr, syncErr, closeOutputErr, - } { - if candidate != nil { - return candidate - } + removeErr := os.Remove(released) + closeErr := lock.file.Close() + if removeErr != nil { + return removeErr } - return nil + return closeErr } -func pathMatchesSHA256(path string, expected [sha256.Size]byte) bool { - status, err := os.Lstat(path) - if err != nil || !status.Mode().IsRegular() { - return false +func attachRuntimeLockReleaseError(result *error, releaseErr error) { + if releaseErr == nil { + return } - actual, err := fileSHA256(path) - return err == nil && actual == expected + if *result == nil { + *result = releaseErr + return + } + *result = fmt.Errorf("%v; runtime lock release failed: %w", *result, releaseErr) } -func installWindowsPairAtomicallyWithVerifier( - srcDir, dstDir string, verifier func(string) error, +func publishRuntimeSetWithRecoveryAndRenamer( + sourceDirectory, destinationDirectory, binaryName, variant string, + verifier func(string) error, + renameFile func(string, string) error, ) (result error) { - names := []string{windowsLauncherName, windowsPayloadName} - if err := verifier(filepath.Join(srcDir, windowsLauncherName)); err != nil { - return fmt.Errorf("source Windows release pair failed verification: %w", err) + sourceNames, ok := runtimeSetNames(sourceDirectory, binaryName, variant) + if !ok { + return fmt.Errorf("source release does not contain a complete runtime set") + } + if verifier != nil { + if err := verifier(filepath.Join(sourceDirectory, binaryName)); err != nil { + return fmt.Errorf("source runtime binary failed verification: %w", err) + } + } + if err := os.MkdirAll(destinationDirectory, 0755); err != nil { + return err + } + if err := requireSafeRuntimeDirectory(destinationDirectory); err != nil { + return err } - lock, err := acquireWindowsPairLock(dstDir) + lock, err := acquireRuntimeSetLock(destinationDirectory) if err != nil { return err } defer func() { - if releaseErr := releaseWindowsPairLock(lock); result == nil && releaseErr != nil { - result = releaseErr - } + attachRuntimeLockReleaseError(&result, releaseRuntimeSetLock(lock)) }() - - // A contender may have committed while this process waited. Preserve that - // complete pair regardless of whether its authenticated release variant is - // byte-identical to this contender. - if windowsPairReady(dstDir, verifier) { - return nil + if err := refreshRuntimeSetLock(lock); err != nil { + return err } - - transaction, err := windowsPairToken() - if err != nil { + if err := reconcileRuntimeBackups( + destinationDirectory, binaryName, variant, verifier, lock, + ); err != nil { return err } - staged := make(map[string]string, len(names)) - stagedDigests := make(map[string][sha256.Size]byte, len(names)) - backups := make(map[string]string, len(names)) - publishedDigests := make(map[string][sha256.Size]byte, len(names)) + // A contender may have committed while this process waited. Its complete + // runtime set wins and is never retired or mixed with this contender. + if runtimeSetReady(destinationDirectory, binaryName, variant, verifier) { + return nil + } + + staged := make(map[string]string, len(sourceNames)) + backupDirectory := "" defer func() { for _, stagedPath := range staged { _ = os.Remove(stagedPath) @@ -838,23 +2023,49 @@ func installWindowsPairAtomicallyWithVerifier( }() operationErr := func() error { - for _, name := range names { - stagePath := filepath.Join( - dstDir, ".cbm-pair-stage-"+transaction+"-"+name) - if err := copyWindowsPairStage( - filepath.Join(srcDir, name), stagePath); err != nil { - return err - } - staged[name] = stagePath - digest, err := fileSHA256(stagePath) + for _, name := range sourceNames { + stagedPath, err := copyRuntimeStage( + filepath.Join(sourceDirectory, name), + destinationDirectory, + name == binaryName, + func() error { return refreshRuntimeSetLock(lock) }, + ) if err != nil { return err } - stagedDigests[name] = digest + staged[name] = stagedPath + } + if err := refreshRuntimeSetLock(lock); err != nil { + return err + } + var err error + backupDirectory, err = createRuntimeBackupDirectory(destinationDirectory) + if err != nil { + return err } - for _, name := range names { - target := filepath.Join(dstDir, name) + entries, err := os.ReadDir(destinationDirectory) + if err != nil { + return err + } + retireNames := []string{binaryName, integrationsFileName} + for _, entry := range entries { + name := entry.Name() + if strings.HasPrefix(name, "cbm-ui-") && strings.HasSuffix(name, ".pack") { + retireNames = append(retireNames, name) + } + } + retired := make(map[string]struct{}, len(retireNames)) + // Retire the executable first so a partial publication is never ready. + for _, name := range retireNames { + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if _, duplicate := retired[name]; duplicate { + continue + } + retired[name] = struct{}{} + target := filepath.Join(destinationDirectory, name) status, err := os.Lstat(target) if os.IsNotExist(err) { continue @@ -862,65 +2073,102 @@ func installWindowsPairAtomicallyWithVerifier( if err != nil { return err } - if !status.Mode().IsRegular() { - return fmt.Errorf( - "refusing unsafe Windows package-cache target: %s", target) + if !status.Mode().IsRegular() || + !platformRuntimeSetFileLinkCountOne(target, status) { + return fmt.Errorf("refusing unsafe package-cache target: %s", target) } - backup := filepath.Join( - dstDir, ".cbm-pair-backup-"+transaction+"-"+name) - if err := os.Rename(target, backup); err != nil { + backup := filepath.Join(backupDirectory, name) + if err := renameFile(target, backup); err != nil { + return err + } + } + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err + } + if runtimeBackupCrashObserver != nil { + if err := runtimeBackupCrashObserver( + runtimeBackupBeforeMarkerEvent, backupDirectory, + ); err != nil { return err } - backups[name] = backup + } + if err := createRuntimeBackupMarker( + backupDirectory, runtimeBackupRetired, + ); err != nil { + return err } - // Payload is the readiness signal and is deliberately published last. - for _, name := range names { - target := filepath.Join(dstDir, name) - if err := os.Rename(staged[name], target); err != nil { + // runtimeSetNames orders every sidecar before the binary readiness signal. + for _, name := range sourceNames { + if err := assertRuntimeSetLockOwner(lock); err != nil { return err } + target := filepath.Join(destinationDirectory, name) + if err := renameFile(staged[name], target); err != nil { + identical, compareErr := filesEqualSHA256(staged[name], target) + if compareErr != nil || !identical { + return err + } + _ = os.Remove(staged[name]) + } delete(staged, name) - publishedDigests[name] = stagedDigests[name] } - if !windowsPairReady(dstDir, verifier) { - return fmt.Errorf("published Windows package-cache pair failed verification") + if !runtimeSetReady(destinationDirectory, binaryName, variant, verifier) { + return fmt.Errorf("published package-cache runtime set failed verification") + } + if err := assertRuntimeSetLockOwner(lock); err != nil { + return err } return nil }() - if operationErr == nil || windowsPairReady(dstDir, verifier) { - for _, backup := range backups { - _ = os.Remove(backup) + if operationErr != nil { + if lockErr := assertRuntimeSetLockOwner(lock); lockErr != nil { + return fmt.Errorf("publication failed and lock ownership changed: %v: %w", operationErr, lockErr) } - return nil } - - // Roll back only files whose digest proves this transaction published them. - for index := len(names) - 1; index >= 0; index-- { - name := names[index] - digest, ok := publishedDigests[name] - target := filepath.Join(dstDir, name) - if ok && pathMatchesSHA256(target, digest) { - _ = os.Remove(target) + if operationErr == nil { + backup, err := inspectRuntimeBackupDirectory( + destinationDirectory, filepath.Base(backupDirectory), binaryName, + ) + if err != nil { + return err } - } - for _, name := range names { - backup, ok := backups[name] - if !ok { - continue + if err := cleanupRuntimeBackup(backup, lock); err != nil { + return err } - target := filepath.Join(dstDir, name) - if _, err := os.Lstat(target); os.IsNotExist(err) { - _ = os.Rename(backup, target) + return nil + } + preserveWinner := runtimeSetReady( + destinationDirectory, binaryName, variant, verifier, + ) + if backupDirectory != "" { + if recoveryErr := reconcileRuntimeBackups( + destinationDirectory, binaryName, variant, verifier, lock, + ); recoveryErr != nil { + return fmt.Errorf( + "%v; package-cache recovery failed: %w", operationErr, recoveryErr, + ) } } + if preserveWinner { + return nil + } return operationErr } -func installWindowsPairAtomically(srcDir, dstDir string) error { - return installWindowsPairAtomicallyWithVerifier( - srcDir, dstDir, verifyCandidate) +func publishRuntimeSetWithRecovery( + sourceDirectory, destinationDirectory, binaryName, variant string, + verifier func(string) error, +) error { + return publishRuntimeSetWithRecoveryAndRenamer( + sourceDirectory, + destinationDirectory, + binaryName, + variant, + verifier, + os.Rename, + ) } func execBinary(executable string, args []string) error { @@ -933,3 +2181,164 @@ func execBinary(executable string, args []string) error { } return syscall.Exec(executable, append([]string{executable}, args...), os.Environ()) } + +func createMutationRuntimeSnapshot( + executable, variant string, + verifier func(string) error, + maintainLease func() error, +) (snapshotDirectory, snapshotExecutable string, result error) { + sourceDirectory := filepath.Dir(executable) + binaryName := filepath.Base(executable) + names, ok := runtimeSetNames(sourceDirectory, binaryName, variant) + if !ok { + return "", "", fmt.Errorf( + "cached runtime assets changed before mutation snapshot", + ) + } + + snapshotDirectory, err := platformCreatePrivateMutationRuntimeDirectory() + if err != nil { + return "", "", err + } + createdDirectory := snapshotDirectory + defer func() { + if result != nil { + _ = os.RemoveAll(createdDirectory) + } + }() + if err := requireSafeRuntimeDirectory(snapshotDirectory); err != nil { + return "", "", fmt.Errorf( + "owner-private mutation runtime snapshot is unsafe: %w", err, + ) + } + + for _, name := range names { + sourcePath := filepath.Join(sourceDirectory, name) + expectedDigest, err := fileSHA256(sourcePath) + if err != nil { + return "", "", err + } + stagedPath, err := copyRuntimeStage( + sourcePath, + snapshotDirectory, + name == binaryName, + maintainLease, + ) + if err != nil { + return "", "", err + } + stagedDigest, digestErr := fileSHA256(stagedPath) + if digestErr != nil || stagedDigest != expectedDigest || + !pathMatchesSHA256(sourcePath, expectedDigest) { + _ = os.Remove(stagedPath) + return "", "", fmt.Errorf( + "cached runtime asset changed while mutation snapshot was copied: %s", + name, + ) + } + if err := os.Rename( + stagedPath, filepath.Join(snapshotDirectory, name), + ); err != nil { + _ = os.Remove(stagedPath) + return "", "", err + } + } + + snapshotExecutable = filepath.Join(snapshotDirectory, binaryName) + if !runtimeSetReady( + snapshotDirectory, binaryName, variant, verifier, + ) { + return "", "", fmt.Errorf( + "private mutation runtime snapshot failed verification", + ) + } + return snapshotDirectory, snapshotExecutable, nil +} + +func runMutationBinary(executable string, args []string) error { + cmd := exec.Command(executable, args...) + cmd.Env = mutationSnapshotEnvironment() + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func execBinaryWithRuntimeLockAndRunner( + executable string, + args []string, + variant string, + verifier func(string) error, + runner func(string, []string) error, +) (result error) { + if runner == nil { + return fmt.Errorf("missing mutation runtime runner") + } + directory := filepath.Dir(executable) + if target, ok := mutationTargetDirectory(args); ok && + pathsReferToSameDirectory(target, directory) { + return fmt.Errorf( + "refusing mutation target that overlaps the shared package cache: %s", + target, + ) + } + if err := requireSafeRuntimeDirectory(directory); err != nil { + return err + } + lock, err := acquireRuntimeSetLock(directory) + if err != nil { + return err + } + lockHeld := true + defer func() { + if lockHeld { + attachRuntimeLockReleaseError(&result, releaseRuntimeSetLock(lock)) + } + }() + if err := refreshRuntimeSetLock(lock); err != nil { + return err + } + if !runtimeSetReady( + directory, filepath.Base(executable), variant, verifier, + ) { + return fmt.Errorf("cached runtime assets changed before mutation launch") + } + snapshotDirectory, snapshotExecutable, err := createMutationRuntimeSnapshot( + executable, + variant, + verifier, + func() error { return refreshRuntimeSetLock(lock) }, + ) + if err != nil { + return err + } + defer func() { + // A hard-killed wrapper can bypass this cleanup while its native child + // still consumes adjacent snapshot assets. Do not scavenge such a + // directory by PID or age, and never replace the native command result + // merely because best-effort cleanup leaked an isolated private copy. + if cleanupErr := runtimeMutationSnapshotCleanup(snapshotDirectory); cleanupErr != nil { + fmt.Fprintf( + os.Stderr, + "codebase-memory-mcp: warning: private mutation snapshot cleanup failed: %v\n", + cleanupErr, + ) + } + }() + releaseErr := releaseRuntimeSetLock(lock) + lockHeld = false + if releaseErr != nil { + return releaseErr + } + return runner(snapshotExecutable, args) +} + +func execBinaryWithRuntimeLock(executable string, args []string) error { + return execBinaryWithRuntimeLockAndRunner( + executable, + args, + runtimeVariant(), + verifyCandidate, + runMutationBinary, + ) +} diff --git a/pkg/go/cmd/codebase-memory-mcp/main_test.go b/pkg/go/cmd/codebase-memory-mcp/main_test.go index 083a924791..c362e8f65f 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/main_test.go @@ -1,197 +1,2149 @@ package main import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" "fmt" + "io" + "net/http" "os" + "os/exec" "path/filepath" + "reflect" + "runtime" + "strconv" "strings" "sync" "testing" + "time" ) -func TestExecutionPathForOSUsesAdjacentWindowsLauncher(t *testing.T) { - payload := filepath.Join("cache", version, windowsPayloadName) - want := filepath.Join("cache", version, windowsLauncherName) +type archiveTestRoundTripper func(*http.Request) (*http.Response, error) - if got := executionPathForOS(payload, "windows"); got != want { - t.Fatalf("Windows execution path = %q, want adjacent launcher %q", got, want) +func (transport archiveTestRoundTripper) RoundTrip( + request *http.Request, +) (*http.Response, error) { + return transport(request) +} + +type archiveCountingBody struct { + reader io.Reader + read int64 +} + +func (body *archiveCountingBody) Read(buffer []byte) (int, error) { + count, err := body.reader.Read(buffer) + body.read += int64(count) + return count, err +} + +func (*archiveCountingBody) Close() error { return nil } + +func TestStandardAndUICachePathsDoNotCollide(t *testing.T) { + t.Setenv("CBM_CACHE_DIR", t.TempDir()) + t.Setenv("CBM_VARIANT", "standard") + standard := binPath() + t.Setenv("CBM_VARIANT", "ui") + ui := binPath() + if standard == ui { + t.Fatalf("standard and UI cache paths collide at %q", standard) + } + if filepath.Base(filepath.Dir(standard)) != "standard" || + filepath.Base(filepath.Dir(ui)) != "ui" { + t.Fatalf("variant cache paths = %q and %q", standard, ui) } - if got := executionPathForOS(payload, "linux"); got != payload { - t.Fatalf("non-Windows execution path = %q, want payload %q", got, payload) +} + +func TestWindowsUsesOneDirectBinary(t *testing.T) { + binary := filepath.Join("cache", version, "ui", windowsBinaryName) + if got := executionPathForOS(binary, "windows"); got != binary { + t.Fatalf("Windows execution path = %q, want direct binary %q", got, binary) + } + if got := binaryNameForOS("windows"); got != "codebase-memory-mcp.exe" { + t.Fatalf("Windows binary name = %q", got) + } + source, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + for _, obsolete := range []string{"payload", "launcher"} { + if strings.Contains(strings.ToLower(string(source)), obsolete) { + t.Fatalf("Go wrapper still contains obsolete %q model", obsolete) + } } } -func TestPortableMutationActionPreservesPackageManagerRefusal(t *testing.T) { +func TestCacheSensitiveMutationActionIsExplicit(t *testing.T) { cases := []struct { args []string want string }{ {args: []string{"update", "--yes"}, want: "update"}, {args: []string{"uninstall", "--yes"}, want: "uninstall"}, - {args: []string{"install", "--yes"}, want: ""}, + {args: []string{"install", "--yes"}, want: "install"}, {args: []string{"cli", "update"}, want: ""}, + {args: []string{"daemon", "update"}, want: ""}, } - for _, testCase := range cases { - if got := portableMutationAction(testCase.args); got != testCase.want { - t.Errorf("portableMutationAction(%q) = %q, want %q", testCase.args, got, testCase.want) + if got := runtimeMutationAction(testCase.args); got != testCase.want { + t.Errorf( + "runtimeMutationAction(%q) = %q, want %q", + testCase.args, + got, + testCase.want, + ) + } + } +} + +func TestWrapperUninstallNeverDefaultsToItsCacheBinary(t *testing.T) { + custom := []string{"uninstall", "--dir", filepath.Join("custom", "bin")} + if got := nativeArgs(custom); !reflect.DeepEqual(got, custom) { + t.Fatalf("custom uninstall args = %q, want %q", got, custom) + } + got := nativeArgs([]string{"uninstall", "--yes"}) + wantBase := "bin" + if runtime.GOOS == "windows" { + wantBase = "codebase-memory-mcp" + } + if len(got) != 4 || got[2] != "--dir" || + filepath.Base(got[3]) != wantBase { + t.Fatalf("default uninstall args do not target managed install: %q", got) + } +} + +func TestArchiveNamespaceIsExactAndUIHasOnePack(t *testing.T) { + binary := "codebase-memory-mcp" + base := archiveNamesForOS("linux", binary) + pack := "cbm-ui-" + strings.Repeat("a", 64) + ".pack" + uiNames := append(append([]string(nil), base...), pack) + got, err := validateArchiveMemberNames(uiNames, base, "ui", false) + if err != nil { + t.Fatal(err) + } + if got != pack { + t.Fatalf("UI pack = %q, want %q", got, pack) + } + badCases := [][]string{ + append(append([]string(nil), uiNames...), "unexpected-root-file"), + append(append([]string(nil), uiNames...), + "cbm-ui-"+strings.Repeat("b", 64)+".pack"), + base, + } + for _, names := range badCases { + if _, err := validateArchiveMemberNames(names, base, "ui", false); err == nil { + t.Fatalf("UI archive namespace accepted invalid members: %q", names) } } + if _, err := validateArchiveMemberNames(uiNames, base, "standard", false); err == nil { + t.Fatal("standard archive accepted a UI pack") + } } -func writeTestWindowsPair(t *testing.T, directory, tag string) { +func writeTarGz(t *testing.T, archivePath string, names []string) { t.Helper() - if err := os.MkdirAll(directory, 0755); err != nil { + file, err := os.Create(archivePath) + if err != nil { t.Fatal(err) } - if err := os.WriteFile( - filepath.Join(directory, windowsLauncherName), - []byte("launcher:"+tag), 0755); err != nil { + gz := gzip.NewWriter(file) + tw := tar.NewWriter(gz) + for _, name := range names { + contents := []byte("contents:" + name) + header := &tar.Header{ + Name: name, + Mode: 0600, + Size: int64(len(contents)), + Typeflag: tar.TypeReg, + } + if err := tw.WriteHeader(header); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(contents); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { t.Fatal(err) } - if err := os.WriteFile( - filepath.Join(directory, windowsPayloadName), - []byte("payload:"+tag), 0755); err != nil { + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { t.Fatal(err) } } -func verifyTestWindowsPair(launcherPath string) error { - launcher, err := os.ReadFile(launcherPath) +func TestTarExtractionValidatesNamespaceAndExtractsRuntimeSet(t *testing.T) { + root := t.TempDir() + archivePath := filepath.Join(root, "release.tar.gz") + destination := filepath.Join(root, "extract") + if err := os.Mkdir(destination, 0755); err != nil { + t.Fatal(err) + } + binary := "codebase-memory-mcp" + archiveNames := archiveNamesForOS("linux", binary) + writeTarGz(t, archivePath, archiveNames) + runtimeNames, err := extractTarGz( + archivePath, + destination, + archiveNames, + []string{binary, integrationsFileName}, + "standard", + ) if err != nil { - return err + t.Fatal(err) + } + want := []string{binary, integrationsFileName} + if !reflect.DeepEqual(runtimeNames, want) { + t.Fatalf("extracted runtime names = %q, want %q", runtimeNames, want) + } + if _, err := os.Stat(filepath.Join(destination, "LICENSE")); !os.IsNotExist(err) { + t.Fatal("non-runtime LICENSE member was extracted") + } + + badArchive := filepath.Join(root, "bad.tar.gz") + writeTarGz(t, badArchive, append(archiveNames, "unexpected-root-file")) + badDestination := filepath.Join(root, "bad-extract") + if err := os.Mkdir(badDestination, 0755); err != nil { + t.Fatal(err) } - payload, err := os.ReadFile( - filepath.Join(filepath.Dir(launcherPath), windowsPayloadName)) + if _, err := extractTarGz( + badArchive, + badDestination, + archiveNames, + []string{binary, integrationsFileName}, + "standard", + ); err == nil { + t.Fatal("tar extraction accepted an unexpected root member") + } +} + +func TestTarExtractionRejectsHardlinkMember(t *testing.T) { + root := t.TempDir() + archivePath := filepath.Join(root, "hardlink.tar.gz") + file, err := os.Create(archivePath) if err != nil { - return err + t.Fatal(err) } - const prefix = "launcher:" - if !strings.HasPrefix(string(launcher), prefix) { - return fmt.Errorf("invalid launcher test fixture") + gz := gzip.NewWriter(file) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{ + Name: "codebase-memory-mcp", Typeflag: tar.TypeLink, + Linkname: "cbm-integrations.json", Mode: 0755, + }); err != nil { + t.Fatal(err) } - tag := strings.TrimPrefix(string(launcher), prefix) - if string(payload) != "payload:"+tag { - return fmt.Errorf("launcher/payload test fixture mismatch") + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + if _, err := tarGzMemberNames(archivePath); err == nil { + t.Fatal("tar namespace accepted a hardlink member") + } +} + +func TestCompressedArchiveDownloadRejectsDeclaredAndActualOverflow(t *testing.T) { + const maxBytes = int64(8) + priorClient := httpsOnlyClient + defer func() { httpsOnlyClient = priorClient }() + + t.Run("declared", func(t *testing.T) { + httpsOnlyClient = &http.Client{Transport: archiveTestRoundTripper( + func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader("")), + ContentLength: maxBytes + 1, + Header: make(http.Header), + Request: request, + }, nil + }, + )} + destination := filepath.Join(t.TempDir(), "release.tar.gz") + err := httpGetWithLimit( + "https://example.invalid/release.tar.gz", destination, maxBytes, + ) + if err == nil || !strings.Contains(err.Error(), "compressed safety limit") { + t.Fatalf("compressed declared overflow error = %v", err) + } + if _, statErr := os.Stat(destination); !os.IsNotExist(statErr) { + t.Fatal("declared oversized archive created a download") + } + }) + + t.Run("actual", func(t *testing.T) { + body := &archiveCountingBody{reader: bytes.NewReader( + bytes.Repeat([]byte("x"), 1024), + )} + httpsOnlyClient = &http.Client{Transport: archiveTestRoundTripper( + func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + ContentLength: -1, + Header: make(http.Header), + Request: request, + }, nil + }, + )} + destination := filepath.Join(t.TempDir(), "release.tar.gz") + err := httpGetWithLimit( + "https://example.invalid/release.tar.gz", destination, maxBytes, + ) + if err == nil || !strings.Contains(err.Error(), "compressed safety limit") { + t.Fatalf("compressed actual overflow error = %v", err) + } + if body.read != maxBytes+1 { + t.Fatalf("oversized response bytes consumed = %d, want %d", body.read, maxBytes+1) + } + if _, statErr := os.Stat(destination); !os.IsNotExist(statErr) { + t.Fatal("rejected compressed archive left a partial download") + } + }) +} + +func testArchiveLimits( + members int, memberBytes, expandedBytes int64, +) archiveResourceLimits { + return archiveResourceLimits{ + compressedBytes: 1024 * 1024, + members: members, + memberBytes: memberBytes, + expandedBytes: expandedBytes, + } +} + +func TestTarArchiveRejectsMemberAndExpandedResourceOverflow(t *testing.T) { + root := t.TempDir() + tests := []struct { + name string + names []string + limits archiveResourceLimits + wantError string + }{ + { + name: "member count", + names: []string{"one", "two", "three"}, + limits: testArchiveLimits(2, 1024, 4096), + wantError: "member safety limit", + }, + { + name: "declared member bytes", + names: []string{"oversized"}, + limits: testArchiveLimits(4, 4, 4096), + wantError: "expanded safety limit", + }, + { + name: "declared aggregate bytes", + names: []string{"one", "two"}, + limits: testArchiveLimits(4, 1024, 15), + wantError: "aggregate expanded safety limit", + }, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + archivePath := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+".tar.gz") + writeTarGz(t, archivePath, testCase.names) + _, err := tarGzMemberNamesWithLimits(archivePath, testCase.limits) + if err == nil || !strings.Contains(err.Error(), testCase.wantError) { + t.Fatalf("tar resource overflow error = %v", err) + } + }) + } +} + +func TestArchiveCopyRejectsActualMemberAggregateAndMetadataMismatch(t *testing.T) { + tests := []struct { + name string + contents string + declared int64 + actual int64 + limits archiveResourceLimits + wantError string + }{ + { + name: "actual member bytes", + contents: "12345", + declared: 5, + limits: testArchiveLimits(1, 4, 10), + wantError: "actual expanded safety limit", + }, + { + name: "actual aggregate bytes", + contents: "12", + declared: 2, + actual: 4, + limits: testArchiveLimits(1, 10, 5), + wantError: "aggregate actual expanded safety limit", + }, + { + name: "declared actual mismatch", + contents: "123", + declared: 2, + limits: testArchiveLimits(1, 10, 10), + wantError: "does not match declared size", + }, + } + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + actual := testCase.actual + var output bytes.Buffer + err := copyArchiveMemberWithLimits( + &output, + strings.NewReader(testCase.contents), + "member", + testCase.declared, + &actual, + testCase.limits, + ) + if err == nil || !strings.Contains(err.Error(), testCase.wantError) { + t.Fatalf("actual archive overflow error = %v", err) + } + }) } - return nil } -func assertTestWindowsPair(t *testing.T, directory, tag string) { +func writeZip(t *testing.T, archivePath string, names []string) { t.Helper() - launcher, err := os.ReadFile(filepath.Join(directory, windowsLauncherName)) + file, err := os.Create(archivePath) if err != nil { t.Fatal(err) } - payload, err := os.ReadFile(filepath.Join(directory, windowsPayloadName)) + zw := zip.NewWriter(file) + for _, name := range names { + member, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := member.Write([]byte("contents:" + name)); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} + +func TestUIZipExtractionIncludesOneContentAddressedPack(t *testing.T) { + root := t.TempDir() + archivePath := filepath.Join(root, "release.zip") + destination := filepath.Join(root, "extract") + if err := os.Mkdir(destination, 0755); err != nil { + t.Fatal(err) + } + pack := "cbm-ui-" + strings.Repeat("a", 64) + ".pack" + archiveNames := archiveNamesForOS("windows", windowsBinaryName) + writeZip(t, archivePath, append(archiveNames, pack)) + runtimeNames, err := extractZip( + archivePath, + destination, + archiveNames, + []string{windowsBinaryName, integrationsFileName}, + "ui", + ) if err != nil { t.Fatal(err) } - if string(launcher) != "launcher:"+tag || string(payload) != "payload:"+tag { - t.Fatalf("pair = %q / %q, want coherent tag %q", launcher, payload, tag) + want := []string{windowsBinaryName, integrationsFileName, pack} + if !reflect.DeepEqual(runtimeNames, want) { + t.Fatalf("extracted runtime names = %q, want %q", runtimeNames, want) + } + for _, name := range want { + if !regularRuntimeFile(filepath.Join(destination, name)) { + t.Fatalf("runtime member %q was not extracted", name) + } } } -func TestWindowsPairPublicationRepairsCorruptAndPartialCaches(t *testing.T) { +func TestZipArchiveRejectsMemberAndExpandedResourceOverflow(t *testing.T) { + root := t.TempDir() tests := []struct { - name string - setup func(t *testing.T, destination string) + name string + names []string + limits archiveResourceLimits + wantError string }{ { - name: "corrupt launcher", - setup: func(t *testing.T, destination string) { - writeTestWindowsPair(t, destination, "old") - if err := os.WriteFile( - filepath.Join(destination, windowsLauncherName), - []byte("corrupt"), 0755); err != nil { - t.Fatal(err) - } - }, + name: "member count", + names: []string{"one", "two", "three"}, + limits: testArchiveLimits(2, 1024, 4096), + wantError: "member safety limit", }, { - name: "corrupt payload", - setup: func(t *testing.T, destination string) { - writeTestWindowsPair(t, destination, "old") - if err := os.WriteFile( - filepath.Join(destination, windowsPayloadName), - []byte("corrupt"), 0755); err != nil { - t.Fatal(err) - } - }, + name: "declared member bytes", + names: []string{"oversized"}, + limits: testArchiveLimits(4, 4, 4096), + wantError: "expanded safety limit", }, { - name: "launcher only", - setup: func(t *testing.T, destination string) { - if err := os.MkdirAll(destination, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile( - filepath.Join(destination, windowsLauncherName), - []byte("launcher:partial"), 0755); err != nil { - t.Fatal(err) - } - }, + name: "declared aggregate bytes", + names: []string{"one", "two"}, + limits: testArchiveLimits(4, 1024, 15), + wantError: "aggregate expanded safety limit", }, } - for _, testCase := range tests { t.Run(testCase.name, func(t *testing.T) { - root := t.TempDir() - source := filepath.Join(root, "source") - destination := filepath.Join(root, "destination") - writeTestWindowsPair(t, source, "candidate") - testCase.setup(t, destination) - - if err := installWindowsPairAtomicallyWithVerifier( - source, destination, verifyTestWindowsPair); err != nil { + archivePath := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+".zip") + writeZip(t, archivePath, testCase.names) + destination := filepath.Join(root, strings.ReplaceAll(testCase.name, " ", "-")+"-extract") + if err := os.Mkdir(destination, 0755); err != nil { t.Fatal(err) } - assertTestWindowsPair(t, destination, "candidate") + _, err := extractZipWithLimits( + archivePath, + destination, + testCase.names, + testCase.names[:1], + "standard", + testCase.limits, + ) + if err == nil || !strings.Contains(err.Error(), testCase.wantError) { + t.Fatalf("zip resource overflow error = %v", err) + } }) } } -func TestWindowsPairPublicationSerializesConcurrentCandidates(t *testing.T) { - root := t.TempDir() - destination := filepath.Join(root, "destination") - if err := os.MkdirAll(destination, 0755); err != nil { - t.Fatal(err) - } - sources := []string{filepath.Join(root, "source-a"), filepath.Join(root, "source-b")} - writeTestWindowsPair(t, sources[0], "a") - writeTestWindowsPair(t, sources[1], "b") - - start := make(chan struct{}) - errors := make(chan error, len(sources)) - var wait sync.WaitGroup - for _, source := range sources { - wait.Add(1) - go func(candidate string) { - defer wait.Done() - <-start - errors <- installWindowsPairAtomicallyWithVerifier( - candidate, destination, verifyTestWindowsPair) - }(source) - } - close(start) - wait.Wait() - close(errors) - for err := range errors { +func writeTestRuntimeSet( + t *testing.T, + directory, binaryName, tag, variant string, +) string { + t.Helper() + if err := os.MkdirAll(directory, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(directory, binaryName), []byte("binary:"+tag), 0755, + ); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(directory, integrationsFileName), + []byte("integrations:"+tag), + 0644, + ); err != nil { + t.Fatal(err) + } + pack := "" + if variant == "ui" { + packContents := []byte("pack:" + tag) + digest := sha256.Sum256(packContents) + pack = "cbm-ui-" + hex.EncodeToString(digest[:]) + ".pack" + if err := os.WriteFile( + filepath.Join(directory, pack), packContents, 0644, + ); err != nil { + t.Fatal(err) + } + } + return pack +} + +func verifyTestBinary(path string) error { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + if !strings.HasPrefix(string(contents), "binary:") { + return fmt.Errorf("invalid test binary") + } + return nil +} + +func TestRuntimeReadinessRequiresTheCompleteSelectedVariant(t *testing.T) { + binary := "codebase-memory-mcp" + standard := filepath.Join(t.TempDir(), "standard") + writeTestRuntimeSet(t, standard, binary, "standard", "standard") + if !runtimeSetReady(standard, binary, "standard", verifyTestBinary) { + t.Fatal("complete standard runtime set is not ready") + } + if err := os.Remove(filepath.Join(standard, integrationsFileName)); err != nil { + t.Fatal(err) + } + if runtimeSetReady(standard, binary, "standard", verifyTestBinary) { + t.Fatal("runtime set without integrations sidecar is ready") + } + + ui := filepath.Join(t.TempDir(), "ui") + writeTestRuntimeSet(t, ui, binary, "ui", "ui") + if !runtimeSetReady(ui, binary, "ui", verifyTestBinary) { + t.Fatal("complete UI runtime set is not ready") + } + uiNames, ok := runtimeSetNames(ui, binary, "ui") + if !ok { + t.Fatal("complete UI runtime set has no member list") + } + pack := uiNames[1] + if err := os.WriteFile(filepath.Join(ui, pack), []byte("corrupt"), 0644); err != nil { + t.Fatal(err) + } + if runtimeSetReady(ui, binary, "ui", verifyTestBinary) { + t.Fatal("UI runtime set accepts pack bytes that do not match the filename digest") + } + writeTestRuntimeSet(t, ui, binary, "ui", "ui") + secondPack := "cbm-ui-" + strings.Repeat("b", 64) + ".pack" + if err := os.WriteFile(filepath.Join(ui, secondPack), []byte("extra"), 0644); err != nil { + t.Fatal(err) + } + if runtimeSetReady(ui, binary, "ui", verifyTestBinary) { + t.Fatal("UI runtime set with two packs is ready") + } +} + +func assertRuntimeTag( + t *testing.T, + directory, binaryName, tag, variant string, +) { + t.Helper() + checks := map[string]string{ + binaryName: "binary:" + tag, + integrationsFileName: "integrations:" + tag, + } + if variant == "ui" { + packContents := []byte("pack:" + tag) + digest := sha256.Sum256(packContents) + checks["cbm-ui-"+hex.EncodeToString(digest[:])+".pack"] = "pack:" + tag + } + for name, want := range checks { + contents, err := os.ReadFile(filepath.Join(directory, name)) if err != nil { t.Fatal(err) } + if string(contents) != want { + t.Fatalf("%s = %q, want %q", name, contents, want) + } } - if err := verifyTestWindowsPair( - filepath.Join(destination, windowsLauncherName)); err != nil { +} + +func TestRuntimePublicationRepairsPartialCache(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + writeTestRuntimeSet(t, source, binary, "candidate", "ui") + if err := os.Mkdir(destination, 0755); err != nil { t.Fatal(err) } - launcher, err := os.ReadFile(filepath.Join(destination, windowsLauncherName)) - if err != nil { + if err := os.WriteFile( + filepath.Join(destination, binary), []byte("corrupt"), 0755, + ); err != nil { + t.Fatal(err) + } + stalePack := "cbm-ui-" + strings.Repeat("b", 64) + ".pack" + if err := os.WriteFile( + filepath.Join(destination, stalePack), []byte("stale"), 0644, + ); err != nil { + t.Fatal(err) + } + + if err := publishRuntimeSetWithRecovery( + source, destination, binary, "ui", verifyTestBinary, + ); err != nil { + t.Fatal(err) + } + assertRuntimeTag(t, destination, binary, "candidate", "ui") + if _, err := os.Stat(filepath.Join(destination, stalePack)); !os.IsNotExist(err) { + t.Fatal("stale UI pack survived runtime publication") + } +} + +func TestRuntimePublicationRepairsDigestMismatchedPack(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + writeTestRuntimeSet(t, source, binary, "candidate", "ui") + oldPack := writeTestRuntimeSet(t, destination, binary, "old", "ui") + if err := os.WriteFile( + filepath.Join(destination, oldPack), []byte("corrupt pack"), 0644, + ); err != nil { + t.Fatal(err) + } + if runtimeSetReady(destination, binary, "ui", verifyTestBinary) { + t.Fatal("digest-mismatched destination pack was accepted before repair") + } + + if err := publishRuntimeSetWithRecovery( + source, destination, binary, "ui", verifyTestBinary, + ); err != nil { + t.Fatal(err) + } + assertRuntimeTag(t, destination, binary, "candidate", "ui") + if _, err := os.Stat(filepath.Join(destination, oldPack)); !os.IsNotExist(err) { + t.Fatal("digest-mismatched UI pack survived runtime publication") + } +} + +func TestRuntimePublicationCommitsSidecarsBeforeBinary(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + pack := writeTestRuntimeSet(t, source, binary, "candidate", "ui") + committed := []string{} + renameFile := func(sourcePath, destinationPath string) error { + name := filepath.Base(destinationPath) + if filepath.Dir(destinationPath) == destination && + (name == binary || name == integrationsFileName || name == pack) { + committed = append(committed, name) + } + return os.Rename(sourcePath, destinationPath) + } + if err := publishRuntimeSetWithRecoveryAndRenamer( + source, + destination, + binary, + "ui", + verifyTestBinary, + renameFile, + ); err != nil { + t.Fatal(err) + } + want := []string{integrationsFileName, pack, binary} + if !reflect.DeepEqual(committed, want) { + t.Fatalf("runtime commit order = %q, want %q", committed, want) + } +} + +func TestRuntimePublicationCrashHelper(t *testing.T) { + if os.Getenv("CBM_TEST_RUNTIME_CRASH_HELPER") != "1" { + return + } + source := os.Getenv("CBM_TEST_RUNTIME_CRASH_SOURCE") + destination := os.Getenv("CBM_TEST_RUNTIME_CRASH_DESTINATION") + marker := os.Getenv("CBM_TEST_RUNTIME_CRASH_MARKER") + crashPhase := os.Getenv("CBM_TEST_RUNTIME_CRASH_PHASE") + binary := "codebase-memory-mcp" + reachCrashGate := func() error { + if err := os.WriteFile(marker, []byte("reached\n"), 0600); err != nil { + return err + } + for { + time.Sleep(time.Hour) + } + } + priorCrashObserver := runtimeBackupCrashObserver + defer func() { runtimeBackupCrashObserver = priorCrashObserver }() + runtimeBackupCrashObserver = func(event, _ string) error { + if event == crashPhase { + return reachCrashGate() + } + return nil + } + renameFile := func(sourcePath, destinationPath string) error { + if err := os.Rename(sourcePath, destinationPath); err != nil { + return err + } + backupParent := filepath.Base(filepath.Dir(destinationPath)) + if crashPhase == "retired-executable" && + runtimeBackupDirectoryName(backupParent) && + filepath.Base(destinationPath) == binary { + return reachCrashGate() + } + if crashPhase == "published-integrations" && + destinationPath == filepath.Join(destination, integrationsFileName) { + return reachCrashGate() + } + if crashPhase == "published-binary" && + destinationPath == filepath.Join(destination, binary) { + return reachCrashGate() + } + return nil + } + if err := publishRuntimeSetWithRecoveryAndRenamer( + source, + destination, + binary, + "standard", + verifyTestBinary, + renameFile, + ); err != nil { t.Fatal(err) } - tag := strings.TrimPrefix(string(launcher), "launcher:") - if tag != "a" && tag != "b" { - t.Fatalf("unexpected concurrent winner tag %q", tag) +} + +func TestKilledRuntimePublisherIsReconciledByLockedReadiness(t *testing.T) { + for _, testCase := range []struct { + name string + crashPhase string + expectedReady bool + expectedBackupMembers int + expectRetiredMarker bool + expectCleanupMarker bool + }{ + { + name: "executable retired before other leaves", + crashPhase: "retired-executable", + expectedReady: false, + expectedBackupMembers: 1, + }, + { + name: "all leaves retired before retirement marker", + crashPhase: runtimeBackupBeforeMarkerEvent, + expectedReady: false, + expectedBackupMembers: 2, + }, + { + name: "partial sidecar publish", + crashPhase: "published-integrations", + expectedReady: false, + expectedBackupMembers: 2, + expectRetiredMarker: true, + }, + { + name: "complete publish before cleanup", + crashPhase: "published-binary", + expectedReady: true, + expectedBackupMembers: 2, + expectRetiredMarker: true, + }, + { + name: "cleanup interrupted after one retired member", + crashPhase: runtimeBackupCleanupRemovedEvent, + expectedReady: true, + expectedBackupMembers: 1, + expectRetiredMarker: true, + expectCleanupMarker: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "destination") + marker := filepath.Join(root, "crash-reached") + binary := "codebase-memory-mcp" + writeTestRuntimeSet(t, source, binary, "candidate", "standard") + writeTestRuntimeSet(t, destination, binary, "old", "standard") + if err := os.WriteFile( + filepath.Join(destination, binary), []byte("corrupt:old"), 0755, + ); err != nil { + t.Fatal(err) + } + + command := exec.Command( + os.Args[0], "-test.run=^TestRuntimePublicationCrashHelper$", + ) + command.Env = append( + os.Environ(), + "CBM_TEST_RUNTIME_CRASH_HELPER=1", + "CBM_TEST_RUNTIME_CRASH_SOURCE="+source, + "CBM_TEST_RUNTIME_CRASH_DESTINATION="+destination, + "CBM_TEST_RUNTIME_CRASH_MARKER="+marker, + "CBM_TEST_RUNTIME_CRASH_PHASE="+testCase.crashPhase, + ) + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + waitResult := make(chan error, 1) + go func() { waitResult <- command.Wait() }() + finished := false + defer func() { + if !finished { + _ = command.Process.Kill() + <-waitResult + } + }() + + deadline := time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(marker); err == nil { + break + } + select { + case err := <-waitResult: + finished = true + t.Fatalf( + "publication helper exited before crash gate: %v: %s", + err, stderr.String(), + ) + default: + } + if time.Now().After(deadline) { + t.Fatalf( + "publication helper did not reach crash gate: %s", + stderr.String(), + ) + } + time.Sleep(10 * time.Millisecond) + } + + entries, err := os.ReadDir(destination) + if err != nil { + t.Fatal(err) + } + backupCount := 0 + for _, entry := range entries { + if runtimeBackupDirectoryName(entry.Name()) { + backupCount++ + if !entry.IsDir() { + t.Fatal("killed publisher backup transaction is not a directory") + } + backupPath := filepath.Join(destination, entry.Name()) + backupEntries, err := os.ReadDir(backupPath) + if err != nil { + t.Fatal(err) + } + memberCount := 0 + for _, backupEntry := range backupEntries { + if runtimeBackupTargetName(backupEntry.Name(), binary) { + memberCount++ + } + } + if memberCount != testCase.expectedBackupMembers { + t.Fatalf( + "backup member count at crash gate = %d, want %d", + memberCount, testCase.expectedBackupMembers, + ) + } + for markerName, expected := range map[string]bool{ + runtimeBackupRetired: testCase.expectRetiredMarker, + runtimeBackupCleanupOnly: testCase.expectCleanupMarker, + } { + _, markerErr := os.Stat(filepath.Join(backupPath, markerName)) + if expected && markerErr != nil { + t.Fatalf("expected backup marker %s is missing: %v", markerName, markerErr) + } + if !expected && !os.IsNotExist(markerErr) { + t.Fatalf("unexpected backup marker %s exists", markerName) + } + } + } + } + if backupCount != 1 { + t.Fatalf("killed publisher backup count = %d, want 1", backupCount) + } + + if err := command.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitResult; err == nil { + t.Fatal("publication helper was not killed") + } + finished = true + + ready, err := runtimeSetReadyLocked( + destination, binary, "standard", verifyTestBinary, + ) + if err != nil { + t.Fatal(err) + } + if ready != testCase.expectedReady { + t.Fatalf( + "reconciled readiness = %v, want %v", + ready, testCase.expectedReady, + ) + } + if !ready { + contents, err := os.ReadFile(filepath.Join(destination, binary)) + if err != nil { + t.Fatal(err) + } + if string(contents) != "corrupt:old" { + t.Fatalf("recovered prior binary = %q", contents) + } + contents, err = os.ReadFile( + filepath.Join(destination, integrationsFileName), + ) + if err != nil { + t.Fatal(err) + } + if string(contents) != "integrations:old" { + t.Fatalf("recovered prior integrations = %q", contents) + } + if err := publishRuntimeSetWithRecovery( + source, destination, binary, "standard", verifyTestBinary, + ); err != nil { + t.Fatal(err) + } + } + assertRuntimeTag( + t, destination, binary, "candidate", "standard", + ) + entries, err = os.ReadDir(destination) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), runtimeBackupPrefix) { + t.Fatalf("orphan backup survived recovery: %s", entry.Name()) + } + } + if _, err := os.Stat(filepath.Join( + destination, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("runtime-set lock survived killed-process recovery") + } + }) + } +} + +func TestConcurrentRuntimePublishersAreSerialized(t *testing.T) { + root := t.TempDir() + firstSource := filepath.Join(root, "source-first") + secondSource := filepath.Join(root, "source-second") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + firstPack := writeTestRuntimeSet( + t, firstSource, binary, "first", "ui", + ) + secondPack := writeTestRuntimeSet( + t, secondSource, binary, "second", "ui", + ) + + firstPaused := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondWaiting := make(chan struct{}) + secondEnteredPublication := make(chan struct{}) + firstResult := make(chan error, 1) + secondResult := make(chan error, 1) + var pauseOnce sync.Once + var waitOnce sync.Once + var enteredOnce sync.Once + priorObserver := runtimeSetLockWaitObserver + runtimeSetLockWaitObserver = func() { + waitOnce.Do(func() { close(secondWaiting) }) + } + defer func() { runtimeSetLockWaitObserver = priorObserver }() + + firstRenamer := func(sourcePath, destinationPath string) error { + if err := os.Rename(sourcePath, destinationPath); err != nil { + return err + } + if destinationPath == filepath.Join(destination, integrationsFileName) { + pauseOnce.Do(func() { + close(firstPaused) + <-releaseFirst + }) + } + return nil + } + secondRenamer := func(sourcePath, destinationPath string) error { + name := filepath.Base(destinationPath) + if filepath.Dir(destinationPath) == destination && + (name == binary || name == integrationsFileName || name == secondPack) { + enteredOnce.Do(func() { close(secondEnteredPublication) }) + } + return os.Rename(sourcePath, destinationPath) + } + + go func() { + firstResult <- publishRuntimeSetWithRecoveryAndRenamer( + firstSource, + destination, + binary, + "ui", + verifyTestBinary, + firstRenamer, + ) + }() + select { + case <-firstPaused: + case <-time.After(5 * time.Second): + t.Fatal("first publisher did not reach the held publication gate") + } + + go func() { + secondResult <- publishRuntimeSetWithRecoveryAndRenamer( + secondSource, + destination, + binary, + "ui", + verifyTestBinary, + secondRenamer, + ) + }() + + concurrentPublication := false + var secondErr error + select { + case <-secondWaiting: + // The serialized implementation reaches this branch while the first + // publisher still owns the held publication gate. + case <-secondEnteredPublication: + concurrentPublication = true + select { + case secondErr = <-secondResult: + case <-time.After(5 * time.Second): + close(releaseFirst) + t.Fatal("concurrent second publisher did not finish") + } + case <-time.After(5 * time.Second): + close(releaseFirst) + t.Fatal("second publisher neither waited nor entered publication") + } + close(releaseFirst) + + var firstErr error + select { + case firstErr = <-firstResult: + case <-time.After(5 * time.Second): + t.Fatal("first publisher did not finish after its gate was released") + } + if !concurrentPublication { + select { + case secondErr = <-secondResult: + case <-time.After(5 * time.Second): + t.Fatal("serialized second publisher did not finish") + } + } + if concurrentPublication { + t.Fatal("second publisher entered publication while the first sequence was held") + } + if firstErr != nil { + t.Fatalf("first publisher failed: %v", firstErr) + } + if secondErr != nil { + t.Fatalf("serialized second publisher failed: %v", secondErr) + } + assertRuntimeTag(t, destination, binary, "first", "ui") + if _, err := os.Stat(filepath.Join(destination, secondPack)); !os.IsNotExist(err) { + t.Fatal("serialized losing publisher damaged the winning runtime set") + } + if _, err := os.Stat(filepath.Join(destination, firstPack)); err != nil { + t.Fatalf("winning UI pack is missing: %v", err) + } +} + +func TestConcurrentPublisherWaitsForRollbackBeforePublishing(t *testing.T) { + root := t.TempDir() + firstSource := filepath.Join(root, "source-first") + secondSource := filepath.Join(root, "source-second") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + firstPack := writeTestRuntimeSet( + t, firstSource, binary, "first", "ui", + ) + secondPack := writeTestRuntimeSet( + t, secondSource, binary, "second", "ui", + ) + oldPack := writeTestRuntimeSet(t, destination, binary, "old", "ui") + verifier := func(path string) error { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + if string(contents) == "binary:old" { + return fmt.Errorf("old runtime set requires repair") + } + return verifyTestBinary(path) + } + + firstPaused := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondWaiting := make(chan struct{}) + secondEnteredPublication := make(chan struct{}) + firstResult := make(chan error, 1) + secondResult := make(chan error, 1) + var pauseOnce sync.Once + var waitOnce sync.Once + var enteredOnce sync.Once + priorObserver := runtimeSetLockWaitObserver + runtimeSetLockWaitObserver = func() { + waitOnce.Do(func() { close(secondWaiting) }) + } + defer func() { runtimeSetLockWaitObserver = priorObserver }() + + failedBinary := false + firstRenamer := func(sourcePath, destinationPath string) error { + if destinationPath == filepath.Join(destination, binary) && !failedBinary { + failedBinary = true + return fmt.Errorf("injected binary publication failure") + } + if err := os.Rename(sourcePath, destinationPath); err != nil { + return err + } + if destinationPath == filepath.Join(destination, integrationsFileName) { + pauseOnce.Do(func() { + close(firstPaused) + <-releaseFirst + }) + } + return nil + } + secondRenamer := func(sourcePath, destinationPath string) error { + name := filepath.Base(destinationPath) + if filepath.Dir(destinationPath) == destination && + (name == binary || name == integrationsFileName || name == secondPack) { + enteredOnce.Do(func() { close(secondEnteredPublication) }) + } + return os.Rename(sourcePath, destinationPath) + } + + go func() { + firstResult <- publishRuntimeSetWithRecoveryAndRenamer( + firstSource, + destination, + binary, + "ui", + verifier, + firstRenamer, + ) + }() + select { + case <-firstPaused: + case <-time.After(5 * time.Second): + t.Fatal("failing publisher did not reach the held publication gate") + } + + go func() { + secondResult <- publishRuntimeSetWithRecoveryAndRenamer( + secondSource, + destination, + binary, + "ui", + verifier, + secondRenamer, + ) + }() + + concurrentPublication := false + var secondErr error + select { + case <-secondWaiting: + case <-secondEnteredPublication: + concurrentPublication = true + select { + case secondErr = <-secondResult: + case <-time.After(5 * time.Second): + close(releaseFirst) + t.Fatal("concurrent second publisher did not finish") + } + case <-time.After(5 * time.Second): + close(releaseFirst) + t.Fatal("second publisher neither waited nor entered failing publication") + } + close(releaseFirst) + + var firstErr error + select { + case firstErr = <-firstResult: + case <-time.After(5 * time.Second): + t.Fatal("failing publisher did not finish rollback") + } + if !concurrentPublication { + select { + case secondErr = <-secondResult: + case <-time.After(5 * time.Second): + t.Fatal("publisher waiting for rollback did not finish") + } + } + if concurrentPublication { + t.Fatal("second publisher entered publication before rollback completed") + } + if firstErr == nil || !failedBinary { + t.Fatal("first publisher did not report the injected publication failure") + } + if secondErr != nil { + t.Fatalf("publisher after rollback failed: %v", secondErr) + } + assertRuntimeTag(t, destination, binary, "second", "ui") + for _, obsoletePack := range []string{firstPack, oldPack} { + if _, err := os.Stat(filepath.Join(destination, obsoletePack)); !os.IsNotExist(err) { + t.Fatalf("obsolete UI pack survived serialized rollback: %s", obsoletePack) + } + } + if _, err := os.Stat(filepath.Join(destination, secondPack)); err != nil { + t.Fatalf("post-rollback winner's UI pack is missing: %v", err) + } + if !runtimeSetReady(destination, binary, "ui", verifier) { + t.Fatal("post-rollback winner is incomplete") + } +} + +func TestRuntimePublicationFailureRestoresPriorCompleteSet(t *testing.T) { + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "destination") + binary := "codebase-memory-mcp" + candidatePack := writeTestRuntimeSet( + t, source, binary, "candidate", "ui", + ) + oldPack := writeTestRuntimeSet(t, destination, binary, "old", "ui") + verifier := func(path string) error { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + if string(contents) == "binary:old" { + return fmt.Errorf("old runtime set requires repair") + } + return verifyTestBinary(path) + } + failed := false + renameFile := func(sourcePath, destinationPath string) error { + if destinationPath == filepath.Join(destination, binary) && !failed { + failed = true + return fmt.Errorf("injected binary publication failure") + } + return os.Rename(sourcePath, destinationPath) + } + + err := publishRuntimeSetWithRecoveryAndRenamer( + source, + destination, + binary, + "ui", + verifier, + renameFile, + ) + if err == nil { + t.Fatal("injected publication failure unexpectedly succeeded") + } + if !failed { + t.Fatal("test did not reach the injected binary publication failure") + } + assertRuntimeTag(t, destination, binary, "old", "ui") + if !runtimeSetReady(destination, binary, "ui", nil) { + t.Fatal("rollback did not restore a structurally complete prior set") + } + if _, err := os.Stat(filepath.Join(destination, candidatePack)); !os.IsNotExist(err) { + t.Fatal("failed candidate UI pack survived rollback") + } + if _, err := os.Stat(filepath.Join(destination, oldPack)); err != nil { + t.Fatalf("rolled-back UI pack is missing: %v", err) + } + if _, err := os.Stat(filepath.Join(destination, runtimeSetLockName)); !os.IsNotExist(err) { + t.Fatal("runtime-set lock survived rollback") + } +} + +func TestMutationSnapshotReleasesCacheLockAndCleansAfterLaunchFailure(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + executable := filepath.Join(directory, binary) + writeTestRuntimeSet(t, directory, binary, "cached", "standard") + var snapshotExecutable string + runnerCalled := false + err := execBinaryWithRuntimeLockAndRunner( + executable, + []string{"install", "--yes"}, + "standard", + verifyTestBinary, + func(candidate string, args []string) error { + runnerCalled = true + snapshotExecutable = candidate + if reflect.DeepEqual(args, []string{"install", "--yes"}) == false { + t.Fatalf("snapshot mutation args = %q", args) + } + if filepath.Dir(candidate) == directory { + t.Fatal("mutation launched from the shared package cache") + } + status, err := os.Lstat(filepath.Dir(candidate)) + if err != nil { + t.Fatal(err) + } + if !status.IsDir() || + (runtime.GOOS != "windows" && status.Mode().Perm()&0077 != 0) { + t.Fatalf("mutation snapshot mode = %v, want owner-private", status.Mode()) + } + assertRuntimeTag( + t, filepath.Dir(candidate), binary, "cached", "standard", + ) + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("cache lock remained held when snapshot runner started") + } + if err := os.WriteFile( + executable, []byte("binary:successor"), 0755, + ); err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(candidate) + if err != nil { + t.Fatal(err) + } + if string(contents) != "binary:cached" { + t.Fatalf("cache mutation changed private snapshot: %q", contents) + } + return fmt.Errorf("injected snapshot launch failure") + }, + ) + if err == nil || !strings.Contains(err.Error(), "injected snapshot launch failure") { + t.Fatalf("mutation launch failure = %v", err) + } + if !runnerCalled { + t.Fatal("verified mutation snapshot was not launched") + } + if _, err := os.Stat(filepath.Dir(snapshotExecutable)); !os.IsNotExist(err) { + t.Fatal("failed mutation left its private runtime snapshot") + } + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("failed mutation left the package-cache lock") + } +} + +func TestMutationSnapshotEnvironmentIgnoresExternalAssetOverrides(t *testing.T) { + t.Setenv("CBM_ASSETS_DIR", filepath.Join(t.TempDir(), "integrations")) + t.Setenv("CBM_UI_ASSETS_DIR", filepath.Join(t.TempDir(), "ui")) + for _, entry := range mutationSnapshotEnvironment() { + name := entry + if separator := strings.IndexByte(entry, '='); separator >= 0 { + name = entry[:separator] + } + if strings.EqualFold(name, "CBM_ASSETS_DIR") || + strings.EqualFold(name, "CBM_UI_ASSETS_DIR") { + t.Fatalf("mutation snapshot environment retained %q", entry) + } + } +} + +func TestMutationSnapshotRejectsExplicitTargetOverlappingCache(t *testing.T) { + directory := t.TempDir() + executable := filepath.Join(directory, "codebase-memory-mcp") + for _, args := range [][]string{ + {"install", "--dir", directory}, + {"uninstall", "--dir=" + directory}, + } { + runnerCalled := false + err := execBinaryWithRuntimeLockAndRunner( + executable, + args, + "standard", + nil, + func(string, []string) error { + runnerCalled = true + return nil + }, + ) + if err == nil || !strings.Contains(err.Error(), "overlaps the shared package cache") { + t.Fatalf("overlapping mutation target error = %v", err) + } + if runnerCalled { + t.Fatal("overlapping cache mutation target was launched") + } + } +} + +func TestMutationSnapshotVerifierFailureNeverLaunches(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + executable := filepath.Join(directory, binary) + writeTestRuntimeSet(t, directory, binary, "cached", "standard") + verifierCalls := 0 + verifier := func(path string) error { + verifierCalls++ + if filepath.Dir(path) != directory { + return fmt.Errorf("injected private snapshot verification failure") + } + return verifyTestBinary(path) + } + runnerCalled := false + err := execBinaryWithRuntimeLockAndRunner( + executable, + []string{"install", "--yes"}, + "standard", + verifier, + func(string, []string) error { + runnerCalled = true + return nil + }, + ) + if err == nil || !strings.Contains(err.Error(), "failed verification") { + t.Fatalf("snapshot verifier failure = %v", err) + } + if verifierCalls != 2 { + t.Fatalf("snapshot verifier calls = %d, want source and snapshot", verifierCalls) + } + if runnerCalled { + t.Fatal("mutation runner was invoked after snapshot verification failure") + } + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("snapshot verification failure left the package-cache lock") + } +} + +func TestMutationSnapshotPreservesCompleteUIRuntimeSet(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + executable := filepath.Join(directory, binary) + pack := writeTestRuntimeSet(t, directory, binary, "ui-cached", "ui") + var snapshotDirectory string + err := execBinaryWithRuntimeLockAndRunner( + executable, + []string{"install", "--yes"}, + "ui", + verifyTestBinary, + func(candidate string, _ []string) error { + snapshotDirectory = filepath.Dir(candidate) + names, ok := runtimeSetNames(snapshotDirectory, binary, "ui") + want := []string{integrationsFileName, pack, binary} + if !ok || !reflect.DeepEqual(names, want) { + t.Fatalf("UI mutation snapshot names = %q, want %q", names, want) + } + assertRuntimeTag( + t, snapshotDirectory, binary, "ui-cached", "ui", + ) + return nil + }, + ) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(snapshotDirectory); !os.IsNotExist(err) { + t.Fatal("successful UI mutation left its private runtime snapshot") + } +} + +func TestMutationSnapshotCleanupFailurePreservesNativeResult(t *testing.T) { + const exitHelper = "CBM_GO_MUTATION_EXIT_HELPER" + if os.Getenv(exitHelper) == "1" { + os.Exit(7) + } + exitCommand := exec.Command( + os.Args[0], + "-test.run=^TestMutationSnapshotCleanupFailurePreservesNativeResult$", + ) + exitCommand.Env = append(os.Environ(), exitHelper+"=1") + nativeErr := exitCommand.Run() + exitErr, ok := nativeErr.(*exec.ExitError) + if !ok { + t.Fatalf("exit helper error = %T %v", nativeErr, nativeErr) + } + + priorCleanup := runtimeMutationSnapshotCleanup + defer func() { runtimeMutationSnapshotCleanup = priorCleanup }() + runtimeMutationSnapshotCleanup = func(path string) error { + if err := os.RemoveAll(path); err != nil { + return err + } + return errors.New("injected private snapshot cleanup failure") + } + for _, testCase := range []struct { + name string + runnerErr error + wantResult error + }{ + {name: "native success remains success"}, + {name: "native exit error identity survives", runnerErr: exitErr, wantResult: exitErr}, + } { + t.Run(testCase.name, func(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + writeTestRuntimeSet(t, directory, binary, "cached", "standard") + result := execBinaryWithRuntimeLockAndRunner( + filepath.Join(directory, binary), + []string{"install", "--yes"}, + "standard", + verifyTestBinary, + func(string, []string) error { return testCase.runnerErr }, + ) + if result != testCase.wantResult { + t.Fatalf("native result identity changed: got %v, want %v", result, testCase.wantResult) + } + if exitErr != nil && testCase.wantResult != nil { + var recovered *exec.ExitError + if !errors.As(result, &recovered) || recovered != exitErr { + t.Fatal("native ExitError was not preserved through cleanup failure") + } + } + }) + } +} + +func TestMutationSnapshotParentDeathLeavesCacheRecoverable(t *testing.T) { + const roleEnvironment = "CBM_GO_MUTATION_SNAPSHOT_ROLE" + role := os.Getenv(roleEnvironment) + if role == "child" { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + info := fmt.Sprintf("%d\n%s\n", os.Getpid(), executable) + if err := os.WriteFile( + os.Getenv("CBM_GO_MUTATION_CHILD_INFO"), []byte(info), 0600, + ); err != nil { + t.Fatal(err) + } + continuePath := os.Getenv("CBM_GO_MUTATION_CHILD_CONTINUE") + deadline := time.Now().Add(20 * time.Second) + for { + if _, err := os.Stat(continuePath); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if !time.Now().Before(deadline) { + t.Fatal("mutation child timed out waiting for successor publication") + } + time.Sleep(10 * time.Millisecond) + } + integrationBytes, err := os.ReadFile(filepath.Join( + filepath.Dir(executable), integrationsFileName, + )) + if err != nil { + t.Fatal(err) + } + if string(integrationBytes) != "integrations:parent-death" { + t.Fatalf("mutation child sidecar changed: %q", integrationBytes) + } + if err := os.WriteFile( + os.Getenv("CBM_GO_MUTATION_CHILD_SURVIVED"), + []byte("survived\n"), + 0600, + ); err != nil { + t.Fatal(err) + } + return + } + if role == "parent" { + directory := os.Getenv("CBM_GO_MUTATION_CACHE") + executable := filepath.Join( + directory, binaryNameForOS(runtime.GOOS), + ) + runner := func(candidate string, args []string) error { + command := exec.Command(candidate, args...) + childEnvironment := make([]string, 0, len(os.Environ())+1) + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, roleEnvironment+"=") { + childEnvironment = append(childEnvironment, entry) + } + } + command.Env = append(childEnvironment, roleEnvironment+"=child") + return command.Run() + } + if err := execBinaryWithRuntimeLockAndRunner( + executable, + []string{ + "-test.run=^TestMutationSnapshotParentDeathLeavesCacheRecoverable$", + }, + "standard", + nil, + runner, + ); err != nil { + t.Fatal(err) + } + t.Fatal("mutation parent returned before the child was released") + } + + root := t.TempDir() + directory := filepath.Join(root, "cache") + if err := os.Mkdir(directory, 0755); err != nil { + t.Fatal(err) + } + binary := binaryNameForOS(runtime.GOOS) + stagedExecutable, err := copyRuntimeStage( + os.Args[0], directory, true, nil, + ) + if err != nil { + t.Fatal(err) + } + cacheExecutable := filepath.Join(directory, binary) + if err := os.Rename(stagedExecutable, cacheExecutable); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(directory, integrationsFileName), + []byte("integrations:parent-death"), + 0644, + ); err != nil { + t.Fatal(err) + } + childInfo := filepath.Join(root, "child-info") + childContinue := filepath.Join(root, "child-continue") + childSurvived := filepath.Join(root, "child-survived") + command := exec.Command( + os.Args[0], + "-test.run=^TestMutationSnapshotParentDeathLeavesCacheRecoverable$", + ) + command.Env = append( + os.Environ(), + roleEnvironment+"=parent", + "CBM_GO_MUTATION_CACHE="+directory, + "CBM_GO_MUTATION_CHILD_INFO="+childInfo, + "CBM_GO_MUTATION_CHILD_CONTINUE="+childContinue, + "CBM_GO_MUTATION_CHILD_SURVIVED="+childSurvived, + ) + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + waitResult := make(chan error, 1) + go func() { waitResult <- command.Wait() }() + parentFinished := false + childPID := 0 + snapshotExecutable := "" + defer func() { + if !parentFinished { + _ = command.Process.Kill() + <-waitResult + } + if childPID > 0 && runtimeSetProcessAlive(childPID) { + if child, err := os.FindProcess(childPID); err == nil { + _ = child.Kill() + } + } + if snapshotExecutable != "" { + _ = os.RemoveAll(filepath.Dir(snapshotExecutable)) + } + }() + + deadline := time.Now().Add(10 * time.Second) + for { + contents, err := os.ReadFile(childInfo) + if err == nil { + lines := strings.Split(strings.TrimSpace(string(contents)), "\n") + if len(lines) != 2 { + t.Fatalf("mutation child info = %q", contents) + } + childPID, err = strconv.Atoi(lines[0]) + if err != nil { + t.Fatal(err) + } + snapshotExecutable = lines[1] + break + } + if !os.IsNotExist(err) { + t.Fatal(err) + } + select { + case err := <-waitResult: + parentFinished = true + t.Fatalf("mutation parent exited before child start: %v: %s", err, stderr.String()) + default: + } + if !time.Now().Before(deadline) { + t.Fatalf("mutation child did not start: %s", stderr.String()) + } + time.Sleep(10 * time.Millisecond) + } + if filepath.Dir(snapshotExecutable) == directory { + t.Fatal("mutation child executed from the shared package cache") + } + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("mutation parent retained the cache lock while child was running") + } + if err := command.Process.Kill(); err != nil { + t.Fatal(err) + } + if err := <-waitResult; err == nil { + t.Fatal("mutation parent was not hard-killed") + } + parentFinished = true + if !runtimeSetProcessAlive(childPID) { + t.Fatal("hard-killing the wrapper also stopped the mutation child") + } + + source := filepath.Join(root, "successor") + writeTestRuntimeSet(t, source, binary, "successor", "standard") + if err := publishRuntimeSetWithRecovery( + source, directory, binary, "standard", verifyTestBinary, + ); err != nil { + t.Fatalf("successor publication after parent death failed: %v", err) + } + assertRuntimeTag(t, directory, binary, "successor", "standard") + if err := os.WriteFile(childContinue, []byte("continue\n"), 0600); err != nil { + t.Fatal(err) + } + deadline = time.Now().Add(10 * time.Second) + for { + if _, err := os.Stat(childSurvived); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if !time.Now().Before(deadline) { + t.Fatal("snapshot child did not survive successor cache publication") + } + time.Sleep(10 * time.Millisecond) + } + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("successor recovery left the package-cache lock") + } +} + +func TestRuntimeSetLockReclaimsOnlyDefinitelyDeadOwner(t *testing.T) { + destination := t.TempDir() + lockPath := filepath.Join(destination, runtimeSetLockName) + if err := os.Mkdir(lockPath, 0700); err != nil { + t.Fatal(err) + } + ownerToken := strings.Repeat("a", runtimeSetLockTokenSize*2) + if err := writeRuntimeSetLockOwner(lockPath, ownerToken); err != nil { + t.Fatal(err) + } + contenderToken := strings.Repeat("b", runtimeSetLockTokenSize*2) + priorProcessAlive := runtimeSetProcessAlive + defer func() { runtimeSetProcessAlive = priorProcessAlive }() + + runtimeSetProcessAlive = func(int) bool { return true } + if runtimeSetTryReclaimLock(lockPath, contenderToken) { + t.Fatal("runtime-set lock reclaimed an owner that may still be live") + } + if _, err := os.Stat(lockPath); err != nil { + t.Fatalf("live owner's runtime-set lock was damaged: %v", err) + } + + runtimeSetProcessAlive = func(int) bool { return false } + if !runtimeSetTryReclaimLock(lockPath, contenderToken) { + t.Fatal("runtime-set lock did not reclaim a definitely dead owner") + } + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Fatal("reclaimed dead-owner runtime-set lock still exists") + } +} + +func TestRuntimeSetLockReclaimsOnlyStaleOwnerlessDirectory(t *testing.T) { + destination := t.TempDir() + lockPath := filepath.Join(destination, runtimeSetLockName) + if err := os.Mkdir(lockPath, 0700); err != nil { + t.Fatal(err) + } + contenderToken := strings.Repeat("c", runtimeSetLockTokenSize*2) + if runtimeSetTryReclaimLock(lockPath, contenderToken) { + t.Fatal("fresh ownerless runtime-set lock was reclaimed") + } + staleTime := time.Now().Add(-runtimeSetOwnerlessStale - time.Second) + if err := os.Chtimes(lockPath, staleTime, staleTime); err != nil { + t.Fatal(err) + } + if !runtimeSetTryReclaimLock(lockPath, contenderToken) { + t.Fatal("stale ownerless runtime-set lock was not reclaimed") + } + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Fatal("reclaimed ownerless runtime-set lock still exists") + } +} + +func TestExpiredLeaseNeverStealsProvenLiveOwner(t *testing.T) { + destination := t.TempDir() + lockPath := filepath.Join(destination, runtimeSetLockName) + record := runtimeSetLockOwnerRecord{ + PID: os.Getpid(), Token: strings.Repeat("a", runtimeSetLockTokenSize*2), + LeaseExpires: time.Now().Add(-time.Second).UnixMilli(), + } + contents, err := json.Marshal(record) + if err != nil { + t.Fatal(err) + } + contents = append(contents, '\n') + if err := os.WriteFile(lockPath, contents, 0600); err != nil { + t.Fatal(err) + } + before, err := os.Lstat(lockPath) + if err != nil { + t.Fatal(err) + } + priorProcessAlive := runtimeSetProcessAlive + defer func() { runtimeSetProcessAlive = priorProcessAlive }() + runtimeSetProcessAlive = func(int) bool { return true } + contender := strings.Repeat("b", runtimeSetLockTokenSize*2) + if runtimeSetTryReclaimLock(lockPath, contender) { + t.Fatal("runtime-set lock stole an expired lease from a proven-live owner") + } + after, err := os.Lstat(lockPath) + if err != nil { + t.Fatalf("live owner's expired lock was removed: %v", err) + } + if !runtimeSetSameLockObject(before, after) { + t.Fatal("live owner's expired lock identity changed") + } + afterContents, err := os.ReadFile(lockPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(afterContents, contents) { + t.Fatalf("live owner's expired lock contents changed: %q", afterContents) + } +} + +func TestLiveLegacyLockIsNotReclaimedSolelyByAge(t *testing.T) { + destination := t.TempDir() + lockPath := filepath.Join(destination, runtimeSetLockName) + if err := os.Mkdir(lockPath, 0700); err != nil { + t.Fatal(err) + } + if err := writeRuntimeSetLockOwner( + lockPath, strings.Repeat("a", runtimeSetLockTokenSize*2), + ); err != nil { + t.Fatal(err) + } + stale := time.Now().Add(-runtimeSetLegacyStale - time.Second) + if err := os.Chtimes(lockPath, stale, stale); err != nil { + t.Fatal(err) + } + priorProcessAlive := runtimeSetProcessAlive + defer func() { runtimeSetProcessAlive = priorProcessAlive }() + runtimeSetProcessAlive = func(int) bool { return true } + if runtimeSetTryReclaimLock( + lockPath, strings.Repeat("b", runtimeSetLockTokenSize*2), + ) { + t.Fatal("live legacy runtime-set lock was reclaimed solely by age") + } + if _, err := os.Stat(lockPath); err != nil { + t.Fatalf("live legacy runtime-set lock was damaged: %v", err) + } +} + +func TestStalledRuntimeLockCreatorNeverDeletesSuccessor(t *testing.T) { + destination := t.TempDir() + var successor *runtimeSetLock + var successorErr error + priorObserver := runtimeSetLockClaimObserver + defer func() { runtimeSetLockClaimObserver = priorObserver }() + runtimeSetLockClaimObserver = func() error { + runtimeSetLockClaimObserver = nil + successor, successorErr = acquireRuntimeSetLock(destination) + return fmt.Errorf("injected stalled creator abort") + } + + if first, err := acquireRuntimeSetLock(destination); err == nil { + _ = releaseRuntimeSetLock(first) + t.Fatal("stalled creator unexpectedly acquired over its successor") + } else if !strings.Contains(err.Error(), "stalled creator abort") { + t.Fatalf("stalled creator failed for wrong reason: %v", err) + } + if successorErr != nil || successor == nil { + t.Fatalf("successor did not acquire reclaimed lock: %v", successorErr) + } + if _, err := os.Stat(successor.path); err != nil { + t.Fatalf("failed creator deleted successor lock: %v", err) + } + if err := releaseRuntimeSetLock(successor); err != nil { + t.Fatal(err) + } +} + +func TestRuntimeReadinessRejectsMultiplyLinkedLeaves(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + writeTestRuntimeSet(t, directory, binary, "linked", "standard") + if err := os.Link( + filepath.Join(directory, integrationsFileName), + filepath.Join(directory, "integrations-hardlink"), + ); err != nil { + t.Skipf("filesystem does not support hard links: %v", err) + } + if runtimeSetReady(directory, binary, "standard", verifyTestBinary) { + t.Fatal("runtime set accepted a multiply-linked integrations leaf") + } +} + +func TestWindowsLongRuntimePathPublishesAndBecomesReady(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Windows long-path regression") + } + root := t.TempDir() + source := filepath.Join(root, "source") + destination := filepath.Join(root, "runtime") + for len(destination) <= 300 { + destination = filepath.Join( + destination, "long-runtime-directory-segment", + ) + } + if len(destination) <= 260 { + t.Fatalf("test runtime path is not long: %d", len(destination)) + } + binary := windowsBinaryName + writeTestRuntimeSet(t, source, binary, "long-path", "standard") + if err := publishRuntimeSetWithRecovery( + source, destination, binary, "standard", verifyTestBinary, + ); err != nil { + t.Fatalf("publication under a long Windows runtime path failed: %v", err) + } + ready, err := runtimeSetReadyLocked( + destination, binary, "standard", verifyTestBinary, + ) + if err != nil { + t.Fatalf("locked readiness under a long Windows runtime path failed: %v", err) + } + if !ready { + t.Fatal("published long-path Windows runtime set was not ready") + } +} + +func TestOrphanReconciliationRejectsMultiplyLinkedBackupMembers(t *testing.T) { + directory := t.TempDir() + binary := "codebase-memory-mcp" + backup := filepath.Join( + directory, runtimeBackupPrefix+strings.Repeat("a", 32), + ) + if err := os.Mkdir(backup, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(backup, runtimeBackupRetired), nil, 0600, + ); err != nil { + t.Fatal(err) + } + member := filepath.Join(backup, binary) + if err := os.WriteFile(member, []byte("binary:old"), 0755); err != nil { + t.Fatal(err) + } + if err := os.Link( + member, filepath.Join(directory, "backup-hardlink-copy"), + ); err != nil { + t.Skipf("filesystem does not support hard links: %v", err) + } + + ready, err := runtimeSetReadyLocked( + directory, binary, "standard", verifyTestBinary, + ) + if err == nil || !strings.Contains( + err.Error(), "unsafe package-cache backup member", + ) { + t.Fatalf("unsafe orphan reconciliation = (%v, %v)", ready, err) + } + if _, err := os.Stat(backup); err != nil { + t.Fatalf("unsafe backup was mutated: %v", err) + } + if _, err := os.Stat(filepath.Join( + directory, runtimeSetLockName, + )); !os.IsNotExist(err) { + t.Fatal("runtime-set lock survived rejected orphan reconciliation") + } +} + +func TestRuntimeSetLockSerializesProcesses(t *testing.T) { + const helperEnv = "CBM_GO_RUNTIME_LOCK_HELPER" + if os.Getenv(helperEnv) == "1" { + destination := os.Getenv("CBM_GO_RUNTIME_LOCK_DIRECTORY") + waitingPath := os.Getenv("CBM_GO_RUNTIME_LOCK_WAITING") + acquiredPath := os.Getenv("CBM_GO_RUNTIME_LOCK_ACQUIRED") + var waitOnce sync.Once + runtimeSetLockWaitObserver = func() { + waitOnce.Do(func() { + if err := os.WriteFile(waitingPath, []byte("waiting"), 0600); err != nil { + t.Fatal(err) + } + }) + } + lock, err := acquireRuntimeSetLock(destination) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(acquiredPath, []byte("acquired"), 0600); err != nil { + _ = releaseRuntimeSetLock(lock) + t.Fatal(err) + } + if err := releaseRuntimeSetLock(lock); err != nil { + t.Fatal(err) + } + return + } + + destination := t.TempDir() + waitingPath := filepath.Join(t.TempDir(), "waiting") + acquiredPath := filepath.Join(t.TempDir(), "acquired") + lock, err := acquireRuntimeSetLock(destination) + if err != nil { + t.Fatal(err) + } + released := false + defer func() { + if !released { + _ = releaseRuntimeSetLock(lock) + } + }() + + cmd := exec.Command(os.Args[0], "-test.run=^TestRuntimeSetLockSerializesProcesses$") + cmd.Env = append( + os.Environ(), + helperEnv+"=1", + "CBM_GO_RUNTIME_LOCK_DIRECTORY="+destination, + "CBM_GO_RUNTIME_LOCK_WAITING="+waitingPath, + "CBM_GO_RUNTIME_LOCK_ACQUIRED="+acquiredPath, + ) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + waitResult := make(chan error, 1) + go func() { waitResult <- cmd.Wait() }() + childFinished := false + defer func() { + if !childFinished { + _ = cmd.Process.Kill() + <-waitResult + } + }() + + deadline := time.Now().Add(5 * time.Second) + for { + select { + case err := <-waitResult: + childFinished = true + t.Fatalf( + "runtime-set lock helper exited before waiting: %v\nstdout: %s\nstderr: %s", + err, + stdout.String(), + stderr.String(), + ) + default: + } + if _, err := os.Stat(waitingPath); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if !time.Now().Before(deadline) { + _ = cmd.Process.Kill() + <-waitResult + childFinished = true + t.Fatalf( + "child did not report waiting on parent lock\nstdout: %s\nstderr: %s", + stdout.String(), + stderr.String(), + ) + } + time.Sleep(5 * time.Millisecond) + } + if _, err := os.Stat(acquiredPath); err == nil { + t.Fatal("child acquired the runtime-set lock while the parent owned it") + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if err := releaseRuntimeSetLock(lock); err != nil { + t.Fatal(err) + } + released = true + select { + case err := <-waitResult: + childFinished = true + if err != nil { + t.Fatalf( + "runtime-set lock helper failed: %v\nstdout: %s\nstderr: %s", + err, + stdout.String(), + stderr.String(), + ) + } + case <-time.After(5 * time.Second): + _ = cmd.Process.Kill() + <-waitResult + childFinished = true + t.Fatal("child did not acquire the released runtime-set lock") + } + if _, err := os.Stat(acquiredPath); err != nil { + t.Fatalf("child never acquired the released runtime-set lock: %v", err) } - assertTestWindowsPair(t, destination, tag) } diff --git a/pkg/go/cmd/codebase-memory-mcp/pair_lock_other.go b/pkg/go/cmd/codebase-memory-mcp/pair_lock_other.go deleted file mode 100644 index 40dc78fe0c..0000000000 --- a/pkg/go/cmd/codebase-memory-mcp/pair_lock_other.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !windows - -package main - -// Windows pair publication is never invoked by a non-Windows wrapper. Keeping -// the fallback conservative also lets platform-neutral transaction tests run -// without introducing a second process-liveness policy. -func cbmWindowsLockProcessAlive(pid int) bool { - return pid > 0 -} diff --git a/pkg/go/cmd/codebase-memory-mcp/pair_lock_windows.go b/pkg/go/cmd/codebase-memory-mcp/pair_lock_windows.go deleted file mode 100644 index d8bb4ecab4..0000000000 --- a/pkg/go/cmd/codebase-memory-mcp/pair_lock_windows.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build windows - -package main - -import "syscall" - -func cbmWindowsLockProcessAlive(pid int) bool { - if pid <= 0 { - return false - } - // PROCESS_QUERY_LIMITED_INFORMATION is sufficient for an existence probe and - // is available on every Windows version supported by the package wrapper. - handle, err := syscall.OpenProcess(0x1000, false, uint32(pid)) - if err == nil { - _ = syscall.CloseHandle(handle) - return true - } - // ERROR_INVALID_PARAMETER is the documented result for a process identifier - // that no longer exists. Access denial is conservatively treated as live. - return err != syscall.Errno(87) -} diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go new file mode 100644 index 0000000000..d4c8645476 --- /dev/null +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go @@ -0,0 +1,34 @@ +//go:build !darwin && !linux && !windows + +package main + +import ( + "fmt" + "os" +) + +func platformCreatePrivateMutationRuntimeDirectory() (string, error) { + directory, err := os.MkdirTemp("", "codebase-memory-mcp-mutation-*") + if err != nil { + return "", err + } + status, err := os.Lstat(directory) + if err != nil || !status.IsDir() || status.Mode()&os.ModeSymlink != 0 || + status.Mode().Perm()&0077 != 0 { + _ = os.RemoveAll(directory) + return "", fmt.Errorf( + "could not create an owner-private mutation runtime snapshot", + ) + } + return directory, nil +} + +// Unsupported wrapper platforms never reclaim a syntactically valid owner by +// PID. Ownerless stale locks remain recoverable through the generic age rule. +func platformRuntimeSetLockProcessAlive(pid int) bool { + return pid > 0 +} + +func platformRuntimeSetFileLinkCountOne(_ string, _ os.FileInfo) bool { + return true +} diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go new file mode 100644 index 0000000000..0c8158b2d4 --- /dev/null +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go @@ -0,0 +1,40 @@ +//go:build darwin || linux + +package main + +import ( + "fmt" + "os" + "syscall" +) + +func platformCreatePrivateMutationRuntimeDirectory() (string, error) { + directory, err := os.MkdirTemp("", "codebase-memory-mcp-mutation-*") + if err != nil { + return "", err + } + status, err := os.Lstat(directory) + if err != nil || !status.IsDir() || status.Mode()&os.ModeSymlink != 0 || + status.Mode().Perm()&0077 != 0 { + _ = os.RemoveAll(directory) + return "", fmt.Errorf( + "could not create an owner-private mutation runtime snapshot", + ) + } + return directory, nil +} + +func platformRuntimeSetLockProcessAlive(pid int) bool { + if pid <= 0 { + return false + } + err := syscall.Kill(pid, 0) + // Only ESRCH proves the owner is gone. Permission denial and every other + // result are conservatively treated as evidence that it may still be live. + return err != syscall.ESRCH +} + +func platformRuntimeSetFileLinkCountOne(_ string, info os.FileInfo) bool { + status, ok := info.Sys().(*syscall.Stat_t) + return ok && status.Nlink == 1 +} diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go new file mode 100644 index 0000000000..3fa233c5a2 --- /dev/null +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go @@ -0,0 +1,197 @@ +//go:build windows + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "unsafe" +) + +const windowsFilePersistentACLs = 0x00000008 + +var ( + windowsAdvapi32 = syscall.NewLazyDLL("advapi32.dll") + windowsKernel32 = syscall.NewLazyDLL("kernel32.dll") + windowsConvertStringSecurityDescriptor = windowsAdvapi32.NewProc( + "ConvertStringSecurityDescriptorToSecurityDescriptorW", + ) + windowsGetVolumeInformationByHandle = windowsKernel32.NewProc( + "GetVolumeInformationByHandleW", + ) +) + +func windowsExtendedPath(path string) (string, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return "", err + } + if strings.HasPrefix(absolute, `\\?\`) { + return absolute, nil + } + if strings.HasPrefix(absolute, `\\.\`) { + return "", fmt.Errorf("refusing Windows device path: %s", path) + } + if strings.HasPrefix(absolute, `\\`) { + return `\\?\UNC\` + strings.TrimPrefix(absolute, `\\`), nil + } + return `\\?\` + absolute, nil +} + +func windowsPrivateDirectorySecurityDescriptor() (uintptr, error) { + token, err := syscall.OpenCurrentProcessToken() + if err != nil { + return 0, err + } + defer token.Close() + user, err := token.GetTokenUser() + if err != nil { + return 0, err + } + userSID, err := user.User.Sid.String() + if err != nil { + return 0, err + } + // Protected DACL: the explicit current user owns the directory and that + // user, LocalSystem, and administrators have full control. + // Object/container inheritance applies the same policy to every runtime + // leaf created inside the snapshot. + sddl, err := syscall.UTF16PtrFromString( + "O:" + userSID + + "D:P(A;OICI;FA;;;" + userSID + ")" + + "(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)", + ) + if err != nil { + return 0, err + } + var descriptor uintptr + result, _, callErr := windowsConvertStringSecurityDescriptor.Call( + uintptr(unsafe.Pointer(sddl)), + 1, + uintptr(unsafe.Pointer(&descriptor)), + 0, + ) + if result == 0 { + if callErr != nil && callErr != syscall.Errno(0) { + return 0, callErr + } + return 0, syscall.EINVAL + } + return descriptor, nil +} + +func windowsDirectoryVolumePersistsACLs(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + var flags uint32 + result, _, callErr := windowsGetVolumeInformationByHandle.Call( + directory.Fd(), + 0, + 0, + 0, + 0, + uintptr(unsafe.Pointer(&flags)), + 0, + 0, + ) + if result == 0 { + if callErr != nil && callErr != syscall.Errno(0) { + return callErr + } + return syscall.EINVAL + } + if flags&windowsFilePersistentACLs == 0 { + return fmt.Errorf("temporary volume does not persist Windows ACLs") + } + return nil +} + +func platformCreatePrivateMutationRuntimeDirectory() (string, error) { + descriptor, err := windowsPrivateDirectorySecurityDescriptor() + if err != nil { + return "", err + } + defer syscall.LocalFree(syscall.Handle(descriptor)) + attributes := syscall.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(syscall.SecurityAttributes{})), + SecurityDescriptor: descriptor, + } + // The protected child DACL makes the snapshot owner-private on a volume + // with persistent ACLs. It cannot prevent a principal that already holds + // FILE_DELETE_CHILD on a hostile shared TEMP parent from replacing the + // pathname, so the supported boundary remains a normal per-user TEMP root. + for attempt := 0; attempt < 128; attempt++ { + token, err := runtimeSetLockToken() + if err != nil { + return "", err + } + directory := filepath.Join( + os.TempDir(), "codebase-memory-mcp-mutation-"+token, + ) + extended, err := windowsExtendedPath(directory) + if err != nil { + return "", err + } + pointer, err := syscall.UTF16PtrFromString(extended) + if err != nil { + return "", err + } + if err := syscall.CreateDirectory(pointer, &attributes); err != nil { + if err == syscall.ERROR_ALREADY_EXISTS { + continue + } + return "", err + } + if err := windowsDirectoryVolumePersistsACLs(directory); err != nil { + _ = os.RemoveAll(directory) + return "", err + } + return directory, nil + } + return "", fmt.Errorf("could not reserve a private mutation runtime snapshot") +} + +func platformRuntimeSetLockProcessAlive(pid int) bool { + if pid <= 0 { + return false + } + if uint64(pid) > uint64(^uint32(0)) { + // An out-of-range value cannot be probed without truncation, so it is not + // evidence that an otherwise valid owner is dead. + return true + } + // PROCESS_QUERY_LIMITED_INFORMATION is sufficient for an existence probe and + // is available on every Windows version supported by the package wrapper. + handle, err := syscall.OpenProcess(0x1000, false, uint32(pid)) + if err == nil { + _ = syscall.CloseHandle(handle) + return true + } + // ERROR_INVALID_PARAMETER is the documented result for a process identifier + // that no longer exists. Access denial is conservatively treated as live. + return err != syscall.Errno(87) +} + +func platformRuntimeSetFileLinkCountOne(path string, _ os.FileInfo) bool { + // os.Open applies Go's absolute long-path handling before CreateFileW. + // Passing an ordinary UTF-16 path directly to syscall.CreateFile would + // reject otherwise valid custom cache paths beyond MAX_PATH. + file, err := os.Open(path) + if err != nil { + return false + } + defer file.Close() + var information syscall.ByHandleFileInformation + if err := syscall.GetFileInformationByHandle( + syscall.Handle(file.Fd()), &information, + ); err != nil { + return false + } + return information.NumberOfLinks == 1 +} diff --git a/pkg/homebrew/Formula/codebase-memory-mcp.rb b/pkg/homebrew/Formula/codebase-memory-mcp.rb index 98736781c1..715bd00d80 100644 --- a/pkg/homebrew/Formula/codebase-memory-mcp.rb +++ b/pkg/homebrew/Formula/codebase-memory-mcp.rb @@ -28,6 +28,7 @@ class CodebaseMemoryMcp < Formula def install bin.install "codebase-memory-mcp" + (share/"codebase-memory-mcp").install "cbm-integrations.json" # Third-party attribution bundle (present in archives since v0.8.1) doc.install "THIRD_PARTY_NOTICES.md" if File.exist?("THIRD_PARTY_NOTICES.md") end @@ -45,5 +46,6 @@ def caveats test do assert_match "codebase-memory-mcp", shell_output("#{bin}/codebase-memory-mcp --version") + assert_path_exists share/"codebase-memory-mcp/cbm-integrations.json" end end diff --git a/pkg/npm/README.md b/pkg/npm/README.md index e9ef2b468d..c64df22beb 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -5,9 +5,9 @@ [![License](https://img.shields.io/badge/license-MIT-green)](https://github.com/DeusData/codebase-memory-mcp/blob/main/LICENSE) [![Platform](https://img.shields.io/badge/macOS_%7C_Linux_%7C_Windows-supported-lightgrey)](https://github.com/DeusData/codebase-memory-mcp/releases/latest) -**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. Ships as a single static binary — this package downloads and runs it automatically. +**The fastest and most efficient code intelligence engine for AI coding agents.** Full-indexes an average repository in milliseconds, the Linux kernel (28M LOC, 75K files) in 3 minutes. Answers structural queries in under 1ms. This npm wrapper downloads, verifies, and caches the selected native runtime set: the executable, its authenticated integration asset, and—when requested—the content-addressed UI pack. -High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 159 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. Zero dependencies. Plug and play across 43 automatic/conditional client surfaces. +High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-sitter/) AST analysis across 159 languages — producing a persistent knowledge graph of functions, classes, call chains, HTTP routes, and cross-service links. 14 MCP tools. No hosted service or API key; this package requires Node.js to manage and launch the cached native runtime set. Plug and play across 43 automatic/conditional client surfaces. ## Installation @@ -15,7 +15,13 @@ High-quality parsing through [tree-sitter](https://tree-sitter.github.io/tree-si npm install -g codebase-memory-mcp ``` -The binary for your platform is downloaded automatically at install time. Then configure your coding agents: +The standard runtime set for your platform is downloaded automatically at install time. To select the UI runtime set and its verified frontend pack, set `CBM_VARIANT=ui` while installing and when invoking the wrapper: + +```bash +CBM_VARIANT=ui npm install -g codebase-memory-mcp +``` + +Then configure your coding agents: ```bash codebase-memory-mcp install @@ -26,7 +32,7 @@ Restart your agent. Say **"Index this project"** — done. ## Why codebase-memory-mcp - **Extreme indexing speed** — Linux kernel (28M LOC, 75K files) in 3 minutes. RAM-first pipeline with LZ4 compression and in-memory SQLite. -- **Plug and play** — single static binary for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (amd64). No Docker, no runtime dependencies, no API keys. +- **Plug and play** — verified native runtime sets for macOS (arm64/amd64), Linux (arm64/amd64), and Windows (arm64/amd64). No Docker or API keys; Node.js owns package download, cache repair, and launch. - **159 languages** — vendored tree-sitter grammars compiled into the binary. Nothing to install, nothing that breaks. - **120x fewer tokens** — 5 structural queries: ~3,400 tokens vs ~412,000 via file-by-file search. - **43 supported automatic/conditional client surfaces** — `install` configures the appropriate MCP, durable-context, and documented hook surfaces without widening client permissions. @@ -44,7 +50,7 @@ Restart your agent. Say **"Index this project"** — done. |---------|-------------| | macOS | arm64, amd64 | | Linux | arm64, amd64 | -| Windows | amd64 | +| Windows | arm64, amd64 | ## Usage diff --git a/pkg/npm/bin.js b/pkg/npm/bin.js index 8b7d82e095..e467981be4 100644 --- a/pkg/npm/bin.js +++ b/pkg/npm/bin.js @@ -4,30 +4,34 @@ // If the binary is missing (e.g. --ignore-scripts), attempts a one-time download. const path = require('path'); -const fs = require('fs'); -const { spawnSync } = require('child_process'); +const os = require('os'); +const { spawn, spawnSync } = require('child_process'); +const { + acquireRuntimeLock, + refreshRuntimeLock, + releaseRuntimeLock, + runtimeSetReady, + runtimeSetReadyLocked, +} = require('./install.js'); const isWindows = process.platform === 'win32'; -// npm is a portable one-shot wrapper. Every platform ships exactly ONE binary, -// so the cached file is the executed file — there is no launcher pair to keep -// in sync and no adjacent payload to validate. +const variant = (process.env.CBM_VARIANT || '').toLowerCase() === 'ui' + ? 'ui' + : 'standard'; +// npm is a portable one-shot wrapper. Every platform ships exactly one binary, +// which is executed directly beside its authenticated runtime sidecars. const binName = isWindows ? 'codebase-memory-mcp.exe' : 'codebase-memory-mcp'; -const binPath = path.join(__dirname, 'bin', binName); +const binDir = path.join(__dirname, 'bin', variant); +const binPath = path.join(binDir, binName); const executionPath = binPath; -const cacheReady = () => { - if (!fs.existsSync(binPath)) { - return false; - } - if (!isWindows) return true; - // A cached Windows binary that cannot report its own version is not usable. - const probe = spawnSync(binPath, ['--version'], { - stdio: 'ignore', - timeout: 15_000, - windowsHide: true, - }); - return !probe.error && probe.status === 0; -}; +const cacheReady = () => runtimeSetReadyLocked(binDir, binName, variant); +function main() { +const requestedMutation = runtimeMutationAction(process.argv.slice(2)); +if (requestedMutation === 'update') { + printUpdateGuidance(); + process.exit(2); +} if (!cacheReady()) { // Binary missing — try running the install script (handles --ignore-scripts case) process.stderr.write('codebase-memory-mcp: binary not found, downloading...\n'); @@ -43,17 +47,105 @@ if (!cacheReady()) { } } -function portableMutationAction(args) { +function runtimeMutationAction(args) { for (const argument of args) { - if (['cli', 'hook-augment', 'config', 'install', '--help', '-h', '--version'].includes(argument)) { + if (argument === '--help' || argument === '-h' || argument === '--version') { return null; } - if (argument === 'update' || argument === 'uninstall') return argument; + if (argument.startsWith('-')) continue; + if (argument === 'install' || argument === 'update' || argument === 'uninstall') { + return argument; + } + if (argument === 'cli' || argument === 'hook-augment' || argument === 'config') { + return null; + } + return null; } return null; } -const result = spawnSync(executionPath, process.argv.slice(2), { +function defaultManagedInstallDir() { + if (!isWindows) return path.join(os.homedir(), '.local', 'bin'); + const localAppData = process.env.LOCALAPPDATA || + path.join(os.homedir(), 'AppData', 'Local'); + return path.join(localAppData, 'Programs', 'codebase-memory-mcp'); +} + +function nativeArgs(args) { + const result = [...args]; + if (runtimeMutationAction(result) === 'uninstall' && + !result.some((argument) => argument === '--dir' || argument.startsWith('--dir='))) { + result.push('--dir', defaultManagedInstallDir()); + } + return result; +} + +function printUpdateGuidance() { + process.stderr.write( + 'This npm copy is maintained by npm. Update it with ' + + '"npm install codebase-memory-mcp@latest" (add -g for a global install). ' + + 'For a managed standalone installation, run ' + + '"codebase-memory-mcp install --yes".\n', + ); +} + +const forwardedArgs = nativeArgs(process.argv.slice(2)); +const mutation = requestedMutation; + +if (mutation === 'install' || mutation === 'uninstall') { + const lock = acquireRuntimeLock(binDir); + let heartbeatError = null; + try { + refreshRuntimeLock(lock); + if (!runtimeSetReady(binDir, binName, variant)) { + throw new Error('cached runtime assets changed before mutation launch'); + } + } catch (error) { + try { releaseRuntimeLock(lock); } catch (_) { /* preserve first error */ } + process.stderr.write(`codebase-memory-mcp: ${error.message}\n`); + process.exit(1); + } + let child = null; + const heartbeat = setInterval(() => { + try { + refreshRuntimeLock(lock); + } catch (error) { + heartbeatError = error; + if (child) child.kill(); + } + }, 20_000); + child = spawn(executionPath, forwardedArgs, { + stdio: 'inherit', + windowsHide: false, + }); + child.on('error', (error) => { + clearInterval(heartbeat); + try { releaseRuntimeLock(lock); } catch (_) { /* report spawn error */ } + process.stderr.write(`codebase-memory-mcp: ${error.message}\n`); + process.exit(1); + }); + child.on('close', (status) => { + clearInterval(heartbeat); + let releaseError = heartbeatError; + try { releaseRuntimeLock(lock); } catch (error) { releaseError ||= error; } + if (releaseError) { + process.stderr.write(`codebase-memory-mcp: ${releaseError.message}\n`); + process.exit(1); + } + if (isWindows && status !== 0 && mutation === 'uninstall') { + process.stderr.write( + 'This npm Windows copy is portable. Use "npm uninstall codebase-memory-mcp" ' + + 'for package maintenance (add -g for a global install). ' + + 'To create or repair a managed standalone installation, run ' + + '"codebase-memory-mcp install --yes".\n', + ); + } + process.exit(status ?? 0); + }); + return; +} + +const result = spawnSync(executionPath, forwardedArgs, { stdio: 'inherit', windowsHide: false, }); @@ -63,19 +155,7 @@ if (result.error) { process.exit(1); } -const mutation = isWindows && result.status !== 0 - ? portableMutationAction(process.argv.slice(2)) - : null; -if (mutation) { - const packageCommand = mutation === 'update' - ? 'npm install codebase-memory-mcp@latest' - : 'npm uninstall codebase-memory-mcp'; - process.stderr.write( - `This npm Windows copy is portable. Use "${packageCommand}" for package ` + - 'maintenance (add -g for a global install), or run ' + - '"codebase-memory-mcp install --yes" once for a managed install with its ' + - 'own install.ps1 for updates.\n', - ); +process.exit(result.status ?? 0); } -process.exit(result.status ?? 0); +main(); diff --git a/pkg/npm/install.js b/pkg/npm/install.js index 2c3ea9b6c3..892a6d314e 100644 --- a/pkg/npm/install.js +++ b/pkg/npm/install.js @@ -13,28 +13,39 @@ const { pipeline } = require('stream'); const REPO = 'DeusData/codebase-memory-mcp'; const VERSION = require('./package.json').version; -const BIN_DIR = path.join(__dirname, 'bin'); +const BIN_ROOT = path.join(__dirname, 'bin'); const MAX_REDIRECTS = 5; const DOWNLOAD_HOP_TIMEOUT_MS = 120_000; const CANDIDATE_TIMEOUT_MS = 15_000; const MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024; const WINDOWS_BINARY_NAME = 'codebase-memory-mcp.exe'; +const INTEGRATIONS_NAME = 'cbm-integrations.json'; +const UI_PACK_PATTERN = /^cbm-ui-[0-9a-f]{64}\.pack$/; const UNIX_ARCHIVE_NAMES = [ 'codebase-memory-mcp', + INTEGRATIONS_NAME, 'LICENSE', 'install.sh', 'THIRD_PARTY_NOTICES.md', ]; const WINDOWS_ARCHIVE_NAMES = [ WINDOWS_BINARY_NAME, + INTEGRATIONS_NAME, 'LICENSE', 'install.ps1', 'THIRD_PARTY_NOTICES.md', ]; -const WINDOWS_PAIR_LOCK_NAME = '.codebase-memory-mcp-pair.lock'; -const WINDOWS_PAIR_LOCK_WAIT_MS = 45_000; -const WINDOWS_PAIR_OWNERLESS_STALE_MS = 30_000; -const WINDOWS_PAIR_LOCK_POLL_MS = 25; +const RUNTIME_LOCK_NAME = '.codebase-memory-mcp-runtime.lock'; +const RUNTIME_LOCK_WAIT_MS = 45_000; +const RUNTIME_OWNERLESS_STALE_MS = 30_000; +const RUNTIME_LOCK_LEASE_MS = 300_000; +const RUNTIME_LOCK_HEARTBEAT_MS = 20_000; +const RUNTIME_LEGACY_STALE_MS = 3_600_000; +const RUNTIME_LOCK_POLL_MS = 25; +const RUNTIME_BACKUP_PREFIX = '.cbm-runtime-backup-'; +const RUNTIME_BACKUP_DIRECTORY_PATTERN = /^\.cbm-runtime-backup-[0-9a-f]{32}$/; +const RUNTIME_BACKUP_RETIRED_MARKER = '.retirement-complete'; +const RUNTIME_BACKUP_CLEANUP_MARKER = '.cleanup-only'; const SLEEP_WORD = new Int32Array(new SharedArrayBuffer(4)); function getPlatform() { @@ -69,7 +80,7 @@ function validateUrl(rawUrl) { return parsed.href; } -function validateExactTarMemberListing(listing, expectedNames) { +function validateExactTarMemberListing(listing, expectedNames, ui = false) { if (listing.includes('\0')) { throw new Error('tar member listing contains a NUL byte'); } @@ -77,34 +88,61 @@ function validateExactTarMemberListing(listing, expectedNames) { if (members[members.length - 1] === '') members.pop(); const expected = new Set(expectedNames); const seen = new Set(); + let uiPackName = null; for (const member of members) { - if (!member || !expected.has(member) || seen.has(member)) { + const isUiPack = ui && UI_PACK_PATTERN.test(member); + if (!member || (!expected.has(member) && !isUiPack) || seen.has(member) || + (isUiPack && uiPackName !== null)) { throw new Error(`archive contains an unexpected or duplicate root member: ${JSON.stringify(member)}`); } seen.add(member); + if (isUiPack) uiPackName = member; } - if (members.length !== expectedNames.length || seen.size !== expected.size) { + const expectedCount = expectedNames.length + (ui ? 1 : 0); + if (members.length !== expectedCount || seen.size !== expectedCount || + expectedNames.some((name) => !seen.has(name)) || (ui && uiPackName === null)) { throw new Error( - `archive must contain exactly these root files: ${expectedNames.join(', ')}`, + `archive must contain exactly these root files${ui ? ' plus one content-addressed UI pack' : ''}: ${expectedNames.join(', ')}`, ); } + return uiPackName; +} + +function validateRegularTarMemberListing(listing, expectedCount) { + const entries = listing.split(/\r?\n/).filter((line) => line.length > 0); + if (entries.length !== expectedCount || + entries.some((line) => line[0] !== '-')) { + throw new Error('archive members must be independent regular files (no links)'); + } } function extractExactTarArchive( - archivePath, destPath, expectedNames, targetName, runFile = execFileSync, + archivePath, destPath, expectedNames, targetNames, ui = false, + runFile = execFileSync, ) { const listing = runFile('tar', ['-tzf', archivePath], { encoding: 'utf8', maxBuffer: MAX_CHECKSUM_MANIFEST_BYTES, windowsHide: true, }); - validateExactTarMemberListing(listing, expectedNames); - // Extract only the fixed root executable. Even a malformed tar implementation - // cannot write an unvalidated companion member outside the private temp tree. - runFile('tar', ['-xzf', archivePath, '-C', destPath, targetName], { + const uiPackName = validateExactTarMemberListing(listing, expectedNames, ui); + const verboseListing = runFile('tar', ['-tvzf', archivePath], { + encoding: 'utf8', + maxBuffer: MAX_CHECKSUM_MANIFEST_BYTES, + windowsHide: true, + }); + validateRegularTarMemberListing( + verboseListing, expectedNames.length + (ui ? 1 : 0), + ); + const names = Array.isArray(targetNames) ? [...targetNames] : [targetNames]; + if (uiPackName !== null) names.push(uiPackName); + // Extract only the authenticated runtime set. Even a malformed tar + // implementation cannot write an unvalidated non-runtime member. + runFile('tar', ['-xzf', archivePath, '-C', destPath, ...names], { stdio: 'inherit', windowsHide: true, }); + return names; } function downloadHop(url, dest, maxBytes) { @@ -204,7 +242,7 @@ function parseExpectedChecksum(manifest, archiveName) { } function verifyCandidate(candidatePath) { - execFileSync(candidatePath, ['--version'], { + execFileSync(candidatePath, ['--verify-runtime-assets'], { stdio: ['ignore', 'pipe', 'pipe'], timeout: CANDIDATE_TIMEOUT_MS, windowsHide: true, @@ -227,21 +265,101 @@ function fileSha256(candidatePath) { return digest.digest('hex'); } -function filesEqualSha256(leftPath, rightPath) { +function sleepSync(milliseconds) { + Atomics.wait(SLEEP_WORD, 0, 0, milliseconds); +} + +function runtimeVariant(env = process.env) { + return (env.CBM_VARIANT || '').toLowerCase() === 'ui' ? 'ui' : 'standard'; +} + +function cacheDirForVariant(variant) { + return path.join(BIN_ROOT, variant); +} + +function requireSafeRuntimeDirectory(directory) { + const status = fs.lstatSync(directory); + if (!status.isDirectory() || status.isSymbolicLink()) { + throw new Error(`refusing unsafe package-cache directory: ${directory}`); + } +} + +function regularFile(pathname) { try { - const left = fs.statSync(leftPath); - const right = fs.statSync(rightPath); - return left.isFile() - && right.isFile() - && left.size === right.size - && fileSha256(leftPath) === fileSha256(rightPath); + const status = fs.lstatSync(pathname); + return status.isFile() && !status.isSymbolicLink() && status.nlink === 1; } catch (_) { return false; } } -function sleepSync(milliseconds) { - Atomics.wait(SLEEP_WORD, 0, 0, milliseconds); +function uiPackMatchesDigest(directory, name) { + try { + const expected = name.slice('cbm-ui-'.length, -'.pack'.length); + return fileSha256(path.join(directory, name)) === expected; + } catch (_) { + return false; + } +} + +function runtimeSetNames(directory, binaryName, variant) { + if (!regularFile(path.join(directory, binaryName)) || + !regularFile(path.join(directory, INTEGRATIONS_NAME))) { + return null; + } + let packLikeNames; + try { + packLikeNames = fs.readdirSync(directory) + .filter((name) => name.startsWith('cbm-ui-') && name.endsWith('.pack')); + } catch (_) { + return null; + } + const validPackNames = packLikeNames.filter( + (name) => UI_PACK_PATTERN.test(name) && + regularFile(path.join(directory, name)) && + uiPackMatchesDigest(directory, name), + ); + if (variant === 'ui') { + if (packLikeNames.length !== 1 || validPackNames.length !== 1) { + return null; + } + } else if (packLikeNames.length !== 0) { + return null; + } + return [ + INTEGRATIONS_NAME, + ...(variant === 'ui' ? validPackNames : []), + binaryName, + ]; +} + +function runtimeSetReady( + directory, binaryName, variant, verifier = verifyCandidate, +) { + if (runtimeSetNames(directory, binaryName, variant) === null) return false; + try { + verifier(path.join(directory, binaryName)); + return true; + } catch (_) { + return false; + } +} + +function runtimeSetReadyLocked( + directory, binaryName, variant, verifier = verifyCandidate, +) { + fs.mkdirSync(directory, { recursive: true, mode: 0o755 }); + requireSafeRuntimeDirectory(directory); + const lock = acquireRuntimeLock(directory); + try { + refreshRuntimeLock(lock); + reconcileRuntimeBackups( + directory, binaryName, variant, verifier, lock, + ); + return runtimeSetReady(directory, binaryName, variant, verifier); + } finally { + releaseRuntimeLock(lock); + } } function processIsAlive(pid) { @@ -256,33 +374,69 @@ function processIsAlive(pid) { } } -function readBinaryLockOwner(lockPath) { +function lockPathStatus(lockPath) { + try { + return fs.lstatSync(lockPath, { bigint: true }); + } catch (_) { + return null; + } +} + +function sameLockObject(left, right) { + return left !== null && right !== null && + left.dev === right.dev && left.ino === right.ino && + left.isFile() === right.isFile() && left.isDirectory() === right.isDirectory(); +} + +function readRuntimeLockOwner(lockPath) { try { + const status = fs.lstatSync(lockPath); + if (status.isSymbolicLink() || (!status.isFile() && !status.isDirectory())) { + return null; + } + const ownerPath = status.isDirectory() + ? path.join(lockPath, 'owner.json') + : lockPath; const owner = JSON.parse( - fs.readFileSync(path.join(lockPath, 'owner.json'), 'utf8'), + fs.readFileSync(ownerPath, 'utf8'), ); if (!Number.isSafeInteger(owner.pid) || owner.pid <= 0 || typeof owner.token !== 'string' || !/^[0-9a-f]{32}$/.test(owner.token)) { return null; } - return owner; + const leaseExpiresMs = Number.isSafeInteger(owner.lease_expires_ms) + ? owner.lease_expires_ms + : null; + return { ...owner, lease_expires_ms: leaseExpiresMs }; } catch (_) { return null; } } -function tryReclaimBinaryLock(lockPath, contenderToken) { - let reclaim = false; - const owner = readBinaryLockOwner(lockPath); +function restoreDisplacedRuntimeLock(reclaimed, lockPath, status) { + if (!status || !status.isFile()) return; + try { + fs.linkSync(reclaimed, lockPath); + fs.unlinkSync(reclaimed); + } catch (_) { + // A new owner already holds the canonical name. Preserve the displaced + // object for its owner to diagnose; never delete an unverified successor. + } +} + +function tryReclaimRuntimeLock(lockPath, contenderToken) { + const observedStatus = lockPathStatus(lockPath); + if (observedStatus === null) return true; + if (observedStatus.isSymbolicLink() || + (!observedStatus.isFile() && !observedStatus.isDirectory())) { + return false; + } + let reclaim; + const owner = readRuntimeLockOwner(lockPath); if (owner) { reclaim = !processIsAlive(owner.pid); } else { - try { - const age = Date.now() - fs.statSync(lockPath).mtimeMs; - reclaim = age >= WINDOWS_PAIR_OWNERLESS_STALE_MS; - } catch (_) { - return true; // The owner released it between observation and inspection. - } + reclaim = Date.now() - Number(observedStatus.mtimeMs) >= RUNTIME_OWNERLESS_STALE_MS; } if (!reclaim) return false; @@ -293,152 +447,467 @@ function tryReclaimBinaryLock(lockPath, contenderToken) { if (err.code === 'ENOENT') return true; return false; } - fs.rmSync(reclaimed, { recursive: true, force: true }); + const movedStatus = lockPathStatus(reclaimed); + if (!sameLockObject(observedStatus, movedStatus)) { + restoreDisplacedRuntimeLock(reclaimed, lockPath, movedStatus); + return false; + } + const movedOwner = readRuntimeLockOwner(reclaimed); + const stillReclaimable = movedOwner === null + ? Date.now() - Number(movedStatus.mtimeMs) >= RUNTIME_OWNERLESS_STALE_MS + : !processIsAlive(movedOwner.pid); + if (!stillReclaimable) { + restoreDisplacedRuntimeLock(reclaimed, lockPath, movedStatus); + return false; + } + if (movedStatus.isDirectory()) { + // Legacy directory locks are accepted only as real, non-symlink + // directories, then quarantined and removed after identity revalidation. + fs.rmSync(reclaimed, { recursive: true, force: true }); + } else { + fs.unlinkSync(reclaimed); + } return true; } -function acquireWindowsBinaryLock(destDir) { - const lockPath = path.join(destDir, WINDOWS_PAIR_LOCK_NAME); +function writeRuntimeLockOwner(fd, token) { + const record = Buffer.from(`${JSON.stringify({ + pid: process.pid, + token, + lease_expires_ms: Date.now() + RUNTIME_LOCK_LEASE_MS, + })}\n`, 'utf8'); + fs.ftruncateSync(fd, 0); + let offset = 0; + while (offset < record.length) { + offset += fs.writeSync(fd, record, offset, record.length - offset, offset); + } + fs.fsyncSync(fd); +} + +function assertRuntimeLockOwner(lock) { + const descriptorStatus = fs.fstatSync(lock.fd, { bigint: true }); + const canonicalStatus = lockPathStatus(lock.lockPath); + const owner = readRuntimeLockOwner(lock.lockPath); + if (!sameLockObject(descriptorStatus, canonicalStatus) || !owner || + owner.token !== lock.token || owner.pid !== process.pid) { + throw new Error('package-cache publication lock ownership changed'); + } +} + +function refreshRuntimeLock(lock) { + assertRuntimeLockOwner(lock); + writeRuntimeLockOwner(lock.fd, lock.token); + assertRuntimeLockOwner(lock); +} + +function acquireRuntimeLock(destDir, beforeClaimPublish = null) { + const lockPath = path.join(destDir, RUNTIME_LOCK_NAME); const token = crypto.randomBytes(16).toString('hex'); - const deadline = Date.now() + WINDOWS_PAIR_LOCK_WAIT_MS; + const claimPath = `${lockPath}.claim-${token}`; + const deadline = Date.now() + RUNTIME_LOCK_WAIT_MS; for (;;) { + let fd = null; try { - fs.mkdirSync(lockPath, { mode: 0o700 }); + fd = fs.openSync(claimPath, 'wx', 0o600); try { - fs.writeFileSync( - path.join(lockPath, 'owner.json'), - `${JSON.stringify({ pid: process.pid, token })}\n`, - { encoding: 'utf8', flag: 'wx', mode: 0o600 }, - ); + writeRuntimeLockOwner(fd, token); + if (beforeClaimPublish) beforeClaimPublish(lockPath); + fs.linkSync(claimPath, lockPath); + fs.unlinkSync(claimPath); + const descriptorStatus = fs.fstatSync(fd, { bigint: true }); + if (!sameLockObject(descriptorStatus, lockPathStatus(lockPath))) { + throw new Error('package-cache publication lock ownership changed'); + } } catch (err) { - fs.rmSync(lockPath, { recursive: true, force: true }); + fs.closeSync(fd); + fd = null; + try { fs.unlinkSync(claimPath); } catch (_) { /* unique claim is gone */ } throw err; } - return { lockPath, token }; + return { lockPath, token, fd }; } catch (err) { + if (fd !== null) { + try { fs.closeSync(fd); } catch (_) { /* already closed */ } + } if (err.code !== 'EEXIST') throw err; - if (tryReclaimBinaryLock(lockPath, token)) continue; + if (tryReclaimRuntimeLock(lockPath, token)) continue; if (Date.now() >= deadline) { - throw new Error('timed out waiting for Windows package-cache publication lock'); + throw new Error('timed out waiting for package-cache publication lock'); } - sleepSync(WINDOWS_PAIR_LOCK_POLL_MS); + sleepSync(RUNTIME_LOCK_POLL_MS); } } } -function releaseWindowsBinaryLock(lock) { - const owner = readBinaryLockOwner(lock.lockPath); - if (!owner || owner.token !== lock.token || owner.pid !== process.pid) { - throw new Error('Windows package-cache publication lock ownership changed'); +function releaseRuntimeLock(lock) { + try { + assertRuntimeLockOwner(lock); + } catch (error) { + try { fs.closeSync(lock.fd); } catch (_) { /* already closed */ } + throw error; } - fs.unlinkSync(path.join(lock.lockPath, 'owner.json')); - fs.rmdirSync(lock.lockPath); -} - -function windowsBinaryReady(destDir, verifier = verifyCandidate) { - const binary = path.join(destDir, WINDOWS_BINARY_NAME); + const released = `${lock.lockPath}.released-${lock.token}`; try { - const status = fs.lstatSync(binary); - if (!status.isFile() || status.isSymbolicLink()) { - return false; - } - verifier(binary); - return true; - } catch (_) { - return false; + fs.renameSync(lock.lockPath, released); + } catch (error) { + fs.closeSync(lock.fd); + throw error; + } + const descriptorStatus = fs.fstatSync(lock.fd, { bigint: true }); + const releasedStatus = lockPathStatus(released); + const owner = readRuntimeLockOwner(released); + if (!sameLockObject(descriptorStatus, releasedStatus) || !owner || + owner.token !== lock.token || owner.pid !== process.pid) { + restoreDisplacedRuntimeLock(released, lock.lockPath, releasedStatus); + fs.closeSync(lock.fd); + throw new Error('package-cache publication lock ownership changed'); + } + try { + fs.unlinkSync(released); + } finally { + fs.closeSync(lock.fd); } } function pathMatchesDigest(candidatePath, expectedDigest) { try { const status = fs.lstatSync(candidatePath); - return status.isFile() && !status.isSymbolicLink() && + return status.isFile() && !status.isSymbolicLink() && status.nlink === 1 && fileSha256(candidatePath) === expectedDigest; } catch (_) { return false; } } -function removeOwnedPublishedBinary(destDir, publishedDigests) { - for (const name of [WINDOWS_BINARY_NAME]) { - const digest = publishedDigests.get(name); +function copyRuntimeStageWithLease(source, staged, lock) { + const input = fs.openSync(source, 'r'); + let output = null; + const buffer = Buffer.allocUnsafe(1024 * 1024); + let lastRefresh = Date.now(); + try { + output = fs.openSync(staged, 'wx', 0o600); + for (;;) { + const count = fs.readSync(input, buffer, 0, buffer.length, null); + if (count === 0) break; + let offset = 0; + while (offset < count) { + offset += fs.writeSync(output, buffer, offset, count - offset, null); + } + if (Date.now() - lastRefresh >= RUNTIME_LOCK_HEARTBEAT_MS) { + refreshRuntimeLock(lock); + lastRefresh = Date.now(); + } + } + fs.fsyncSync(output); + } catch (error) { + try { fs.unlinkSync(staged); } catch (_) { /* never published */ } + throw error; + } finally { + fs.closeSync(input); + if (output !== null) fs.closeSync(output); + } +} + +// A grouped backup is a tiny crash journal. Before the retired marker, only +// moved members are restored; after it, partial published targets are replaced. +// cleanup-only is written only after the destination is accepted or restored, +// so a second crash can finish deletion without interpreting a partial journal. +function runtimeBackupTargetName(name, binaryName) { + return name === binaryName || name === INTEGRATIONS_NAME || + (name.startsWith('cbm-ui-') && name.endsWith('.pack')); +} + +function createRuntimeBackupDirectory(destDir) { + for (let attempt = 0; attempt < 128; attempt += 1) { + const token = crypto.randomBytes(16).toString('hex'); + const backupDir = path.join(destDir, `${RUNTIME_BACKUP_PREFIX}${token}`); + try { + fs.mkdirSync(backupDir, { mode: 0o700 }); + return backupDir; + } catch (err) { + if (!err || err.code !== 'EEXIST') throw err; + } + } + throw new Error('could not reserve a package-cache backup transaction'); +} + +function createRuntimeBackupMarker(backupDir, markerName) { + const marker = path.join(backupDir, markerName); + const fd = fs.openSync(marker, 'wx', 0o600); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } +} + +function inspectRuntimeBackupDirectory(destDir, entryName, binaryName) { + const backupDir = path.join(destDir, entryName); + const directoryStatus = fs.lstatSync(backupDir); + if (!directoryStatus.isDirectory() || directoryStatus.isSymbolicLink()) { + throw new Error(`refusing unsafe package-cache backup: ${backupDir}`); + } + const backup = { + path: backupDir, + retired: false, + cleanupOnly: false, + files: new Map(), + }; + for (const name of fs.readdirSync(backupDir).sort()) { + const candidate = path.join(backupDir, name); + const status = fs.lstatSync(candidate); + if (name === RUNTIME_BACKUP_RETIRED_MARKER || + name === RUNTIME_BACKUP_CLEANUP_MARKER) { + if (!status.isFile() || status.isSymbolicLink() || + status.nlink !== 1 || status.size !== 0) { + throw new Error(`refusing unsafe package-cache backup marker: ${candidate}`); + } + if (name === RUNTIME_BACKUP_RETIRED_MARKER) backup.retired = true; + if (name === RUNTIME_BACKUP_CLEANUP_MARKER) backup.cleanupOnly = true; + continue; + } + if (!runtimeBackupTargetName(name, binaryName) || + !status.isFile() || status.isSymbolicLink() || status.nlink !== 1) { + throw new Error(`refusing unsafe package-cache backup member: ${candidate}`); + } + backup.files.set(name, { + path: candidate, + digest: fileSha256(candidate), + }); + } + return backup; +} + +function listRuntimeBackupDirectories(destDir, binaryName) { + return fs.readdirSync(destDir) + .filter((name) => RUNTIME_BACKUP_DIRECTORY_PATTERN.test(name)) + .sort() + .map((name) => inspectRuntimeBackupDirectory(destDir, name, binaryName)); +} + +function currentRuntimeTargets(destDir, binaryName) { + const targets = new Map(); + const names = fs.readdirSync(destDir) + .filter((name) => runtimeBackupTargetName(name, binaryName)) + .sort(); + for (const name of names) { const target = path.join(destDir, name); - if (digest && pathMatchesDigest(target, digest)) { - try { fs.unlinkSync(target); } catch (_) { /* recovery continues below */ } + const status = fs.lstatSync(target); + if (!status.isFile() || status.isSymbolicLink() || status.nlink !== 1) { + throw new Error(`refusing unsafe package-cache target: ${target}`); + } + targets.set(name, target); + } + return targets; +} + +function cleanupRuntimeBackup(backup, lock) { + assertRuntimeLockOwner(lock); + if (!backup.cleanupOnly) { + createRuntimeBackupMarker(backup.path, RUNTIME_BACKUP_CLEANUP_MARKER); + backup.cleanupOnly = true; + } + for (const name of [...backup.files.keys()].sort()) { + assertRuntimeLockOwner(lock); + const member = backup.files.get(name); + if (!member || !pathMatchesDigest(member.path, member.digest)) { + throw new Error(`package-cache backup member changed during cleanup: ${name}`); + } + assertRuntimeLockOwner(lock); + fs.unlinkSync(member.path); + } + if (backup.retired) { + assertRuntimeLockOwner(lock); + fs.unlinkSync(path.join(backup.path, RUNTIME_BACKUP_RETIRED_MARKER)); + } + assertRuntimeLockOwner(lock); + fs.unlinkSync(path.join(backup.path, RUNTIME_BACKUP_CLEANUP_MARKER)); + fs.rmdirSync(backup.path); +} + +function copyRuntimeBackupFile(member, target, executable, lock) { + const staged = path.join( + path.dirname(target), + `.cbm-runtime-recovery-${crypto.randomBytes(16).toString('hex')}`, + ); + try { + copyRuntimeStageWithLease(member.path, staged, lock); + fs.chmodSync(staged, executable ? 0o755 : 0o644); + if (fileSha256(staged) !== member.digest) { + throw new Error('package-cache backup changed while it was being restored'); } + assertRuntimeLockOwner(lock); + fs.renameSync(staged, target); + } finally { + try { fs.unlinkSync(staged); } catch (_) { /* renamed or never completed */ } } } -function restoreBinaryBackups(destDir, backups) { - for (const name of [WINDOWS_BINARY_NAME]) { - const backup = backups.get(name); +function reconcileRuntimeBackups( + destDir, binaryName, variant, verifier, lock, +) { + refreshRuntimeLock(lock); + let backups = listRuntimeBackupDirectories(destDir, binaryName); + for (const backup of backups.filter((candidate) => candidate.cleanupOnly)) { + cleanupRuntimeBackup(backup, lock); + } + backups = backups.filter((candidate) => !candidate.cleanupOnly); + if (backups.length === 0) return; + + if (runtimeSetReady(destDir, binaryName, variant, verifier)) { + for (const backup of backups) cleanupRuntimeBackup(backup, lock); + return; + } + if (backups.length === 0) return; + + for (const backup of backups.filter( + (candidate) => !candidate.retired && candidate.files.size === 0, + )) { + cleanupRuntimeBackup(backup, lock); + } + backups = backups.filter( + (candidate) => candidate.retired || candidate.files.size !== 0, + ); + if (backups.length === 0) return; + if (backups.length !== 1) { + throw new Error('multiple package-cache backup transactions require manual recovery'); + } + + const backup = backups[0]; + const targets = currentRuntimeTargets(destDir, binaryName); + if (!backup.retired) { + for (const [name, member] of backup.files.entries()) { + const target = targets.get(name); + if (target && !pathMatchesDigest(target, member.digest)) { + throw new Error(`package-cache backup conflicts with current target: ${name}`); + } + } + } else { + const binaryTarget = targets.get(binaryName); + if (binaryTarget) { + assertRuntimeLockOwner(lock); + fs.unlinkSync(binaryTarget); + targets.delete(binaryName); + } + for (const name of [...targets.keys()].sort()) { + assertRuntimeLockOwner(lock); + fs.unlinkSync(targets.get(name)); + targets.delete(name); + } + } + + const restoreNames = [...backup.files.keys()] + .filter((name) => name !== binaryName) + .sort(); + if (backup.files.has(binaryName)) restoreNames.push(binaryName); + for (const name of restoreNames) { + const member = backup.files.get(name); const target = path.join(destDir, name); - if (!backup || fs.existsSync(target)) continue; - try { fs.renameSync(backup, target); } catch (_) { /* preserve backup */ } + if (!pathMatchesDigest(target, member.digest)) { + copyRuntimeBackupFile(member, target, name === binaryName, lock); + } } + for (const [name, member] of backup.files.entries()) { + if (!pathMatchesDigest(path.join(destDir, name), member.digest)) { + throw new Error(`package-cache backup restoration failed: ${name}`); + } + } + cleanupRuntimeBackup(backup, lock); } -function installWindowsBinaryAtomically(sourceDir, destDir, verifier = verifyCandidate) { +function publishRuntimeSetWithRecovery( + sourceDir, destDir, binaryName, variant, verifier = verifyCandidate, +) { + const sourceNames = runtimeSetNames(sourceDir, binaryName, variant); + if (sourceNames === null) { + throw new Error('source release does not contain a complete runtime set'); + } // Authenticate the source binary before it can contend for the cache. - verifier(path.join(sourceDir, WINDOWS_BINARY_NAME)); - const lock = acquireWindowsBinaryLock(destDir); + verifier(path.join(sourceDir, binaryName)); + fs.mkdirSync(destDir, { recursive: true }); + requireSafeRuntimeDirectory(destDir); + const lock = acquireRuntimeLock(destDir); let operationError = null; try { - // A contender may have completed while this process waited. Its executable - // wins and is never moved or deleted. - if (windowsBinaryReady(destDir, verifier)) return; + // A contender may have completed while this process waited. Its complete + // runtime set wins and is never moved or deleted. + refreshRuntimeLock(lock); + reconcileRuntimeBackups(destDir, binaryName, variant, verifier, lock); + if (runtimeSetReady(destDir, binaryName, variant, verifier)) return; - const transaction = crypto.randomBytes(16).toString('hex'); + const publicationToken = crypto.randomBytes(16).toString('hex'); const stagedPaths = new Map(); - const stagedDigests = new Map(); - const backups = new Map(); - const publishedDigests = new Map(); + let backupDirectory = null; try { - for (const name of [WINDOWS_BINARY_NAME]) { - const staged = path.join(destDir, `.cbm-pair-stage-${transaction}-${name}`); - fs.copyFileSync(path.join(sourceDir, name), staged, fs.constants.COPYFILE_EXCL); - fs.chmodSync(staged, 0o755); + for (const name of sourceNames) { + const staged = path.join(destDir, `.cbm-runtime-stage-${publicationToken}-${name}`); + copyRuntimeStageWithLease(path.join(sourceDir, name), staged, lock); + fs.chmodSync(staged, name === binaryName ? 0o755 : 0o644); stagedPaths.set(name, staged); - stagedDigests.set(name, fileSha256(staged)); } + refreshRuntimeLock(lock); + backupDirectory = createRuntimeBackupDirectory(destDir); - for (const name of [WINDOWS_BINARY_NAME]) { + const stalePackNames = fs.readdirSync(destDir) + .filter((name) => name.startsWith('cbm-ui-') && name.endsWith('.pack')); + const retireNames = [...new Set([ + binaryName, + INTEGRATIONS_NAME, + ...stalePackNames, + ])]; + // Retire the binary first so an incomplete publication is never ready. + for (const name of retireNames) { + assertRuntimeLockOwner(lock); const target = path.join(destDir, name); const status = fs.lstatSync(target, { throwIfNoEntry: false }); if (!status) continue; - if (!status.isFile() || status.isSymbolicLink()) { - throw new Error(`refusing unsafe Windows package-cache target: ${target}`); + if (!status.isFile() || status.isSymbolicLink() || status.nlink !== 1) { + throw new Error(`refusing unsafe package-cache target: ${target}`); } - const backup = path.join(destDir, `.cbm-pair-backup-${transaction}-${name}`); + const backup = path.join(backupDirectory, name); fs.renameSync(target, backup); - backups.set(name, backup); } + assertRuntimeLockOwner(lock); + createRuntimeBackupMarker( + backupDirectory, RUNTIME_BACKUP_RETIRED_MARKER, + ); - for (const name of [WINDOWS_BINARY_NAME]) { + // Sidecars are committed first. The executable is the final readiness + // signal, so no observer can accept a newly published partial set. + for (const name of sourceNames) { + assertRuntimeLockOwner(lock); const target = path.join(destDir, name); fs.renameSync(stagedPaths.get(name), target); stagedPaths.delete(name); - publishedDigests.set(name, stagedDigests.get(name)); - } - if (!windowsBinaryReady(destDir, verifier)) { - throw new Error('published Windows package-cache binary failed verification'); } - for (const backup of backups.values()) { - try { fs.unlinkSync(backup); } catch (_) { /* valid binary is already committed */ } + if (!runtimeSetReady(destDir, binaryName, variant, verifier)) { + throw new Error('published package-cache runtime set failed verification'); } + assertRuntimeLockOwner(lock); + cleanupRuntimeBackup( + inspectRuntimeBackupDirectory( + destDir, path.basename(backupDirectory), binaryName, + ), + lock, + ); return; } catch (err) { - // If a non-cooperating contender nevertheless published a valid binary, - // preserve that winner. Otherwise remove only our exact staged bytes and - // restore the prior files when their target names are still absent. - if (windowsBinaryReady(destDir, verifier)) { - for (const backup of backups.values()) { - try { fs.unlinkSync(backup); } catch (_) { /* winner remains authoritative */ } + assertRuntimeLockOwner(lock); + const preserveWinner = runtimeSetReady( + destDir, binaryName, variant, verifier, + ); + if (backupDirectory !== null) { + try { + reconcileRuntimeBackups( + destDir, binaryName, variant, verifier, lock, + ); + } catch (recoveryError) { + throw new Error( + `${err.message}; package-cache recovery failed: ${recoveryError.message}`, + ); } - return; } - removeOwnedPublishedBinary(destDir, publishedDigests); - restoreBinaryBackups(destDir, backups); + if (preserveWinner) return; throw err; } finally { for (const staged of stagedPaths.values()) { @@ -450,14 +919,16 @@ function installWindowsBinaryAtomically(sourceDir, destDir, verifier = verifyCan throw err; } finally { try { - releaseWindowsBinaryLock(lock); + releaseRuntimeLock(lock); } catch (releaseError) { if (!operationError) throw releaseError; } } } -function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) { +function extractZipOnWindows( + archivePath, destPath, requiredNames, extractNames, ui = false, +) { // -EncodedCommand is a constant program. Paths travel only through the child // environment and are consumed with -LiteralPath, so PowerShell never parses // user/TEMP path bytes as source code or wildcard syntax. @@ -467,6 +938,8 @@ function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) '$zip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', "$seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)", "$requiredNames = @($env:CBM_NPM_REQUIRED_NAMES.Split('|'))", + "$uiRequired = $env:CBM_NPM_UI_REQUIRED -eq '1'", + '$uiPackName = $null', '$targetCounts = @{}', 'foreach ($requiredName in $requiredNames) { $targetCounts[$requiredName] = 0 }', 'try { foreach ($entry in $zip.Entries) { ' + @@ -474,23 +947,36 @@ function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) "$directory = $name.EndsWith('/'); " + "$segmentsPath = if ($directory) { $name.TrimEnd('/') } else { $name }; " + "$segments = @($segmentsPath.Split('/')); " + + '$unixType = (($entry.ExternalAttributes -shr 16) -band 0xF000); ' + "if ([string]::IsNullOrEmpty($segmentsPath) -or $name.StartsWith('/') -or " + "$name.Contains(':') -or $segments -contains '' -or " + "$segments -contains '.' -or $segments -contains '..' -or " + - "@($segments | Where-Object { $_.EndsWith('.') -or $_.EndsWith(' ') }).Count -gt 0) { " + + "@($segments | Where-Object { $_.EndsWith('.') -or $_.EndsWith(' ') }).Count -gt 0 -or " + + '($unixType -ne 0 -and $unixType -ne 0x8000)) { ' + "throw \"unsafe zip entry path: $($entry.FullName)\" }; " + 'if (-not $seen.Add($segmentsPath)) { ' + "throw \"duplicate or case-conflicting zip entry: $($entry.FullName)\" }; " + + "$isUiPack = $uiRequired -and ($name -cmatch '^cbm-ui-[0-9a-f]{64}\\.pack$'); " + + '$isRequired = $false; ' + 'foreach ($requiredName in $requiredNames) { ' + 'if ($name -ceq $requiredName) { ' + - '$targetCounts[$requiredName] = $targetCounts[$requiredName] + 1 } } ' + + '$isRequired = $true; ' + + '$targetCounts[$requiredName] = $targetCounts[$requiredName] + 1 } }; ' + + 'if ($isUiPack) { if ($null -ne $uiPackName) { ' + + 'throw "archive contains more than one UI pack" }; $uiPackName = $name }; ' + + 'if ($directory -or (-not $isRequired -and -not $isUiPack)) { ' + + 'throw "archive contains an unexpected root member: $($entry.FullName)" } ' + '} } finally { $zip.Dispose() }', 'foreach ($requiredName in $requiredNames) { ' + 'if ($targetCounts[$requiredName] -ne 1) { ' + 'throw "archive must contain exactly one $requiredName" } }', - 'if ($seen.Count -ne $requiredNames.Count) { ' + + 'if ($uiRequired -and $null -eq $uiPackName) { ' + + 'throw "archive must contain exactly one content-addressed UI pack" }', + '$expectedCount = $requiredNames.Count; if ($uiRequired) { $expectedCount++ }', + 'if ($seen.Count -ne $expectedCount) { ' + 'throw "archive does not match the exact release root-file allowlist" }', "$extractNames = @($env:CBM_NPM_EXTRACT_NAMES.Split('|'))", + 'if ($uiRequired) { $extractNames += $uiPackName }', '$extractZip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', 'try { foreach ($extractName in $extractNames) { ' + '$entry = @($extractZip.Entries | Where-Object { $_.FullName -ceq $extractName })[0]; ' + @@ -508,6 +994,7 @@ function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) CBM_NPM_DEST_PATH: destPath, CBM_NPM_REQUIRED_NAMES: requiredNames.join('|'), CBM_NPM_EXTRACT_NAMES: extractNames.join('|'), + CBM_NPM_UI_REQUIRED: ui ? '1' : '0', }, stdio: 'inherit', windowsHide: true, @@ -546,43 +1033,29 @@ async function main() { const platform = getPlatform(); const arch = getArch(); const ext = platform === 'windows' ? 'zip' : 'tar.gz'; + const selectedVariant = runtimeVariant(); + const cacheDir = cacheDirForVariant(selectedVariant); // Package-manager shims are portable one-shot instances: one cached binary // per platform, entered directly. const binName = platform === 'windows' ? WINDOWS_BINARY_NAME : 'codebase-memory-mcp'; - const binPath = path.join(BIN_DIR, binName); - const cacheNames = platform === 'windows' - ? [WINDOWS_BINARY_NAME] - : [binName]; - const extractedNames = platform === 'windows' - ? [WINDOWS_BINARY_NAME] - : [binName]; + const extractedNames = [binName, INTEGRATIONS_NAME]; const archiveNames = platform === 'windows' ? WINDOWS_ARCHIVE_NAMES : UNIX_ARCHIVE_NAMES; - const cachePaths = cacheNames.map((name) => path.join(BIN_DIR, name)); - if (platform === 'windows' && windowsBinaryReady(BIN_DIR)) return; - if (platform !== 'windows' && - cachePaths.every((candidate) => fs.existsSync(candidate))) { - try { - verifyCandidate(binPath); - return; // already installed and runnable, nothing to do - } catch (_) { - // Fall through and atomically replace the invalid binary. - } - } + if (runtimeSetReadyLocked(cacheDir, binName, selectedVariant)) return; - fs.mkdirSync(BIN_DIR, { recursive: true }); + fs.mkdirSync(cacheDir, { recursive: true }); // Linux ships a fully-static "-portable" build; the standard linux binary // dynamically links glibc 2.38+ and fails on older distros. macOS/Windows // have no such variant. Keep in sync with install.sh / pypi _cli.py / cli.c. const variant = platform === 'linux' ? '-portable' : ''; - // Opt into the UI build (embedded graph visualization) with CBM_VARIANT=ui. - // Default is the standard (headless) build. Mirrors install.sh --ui. - const ui = (process.env.CBM_VARIANT || '').toLowerCase() === 'ui' ? 'ui-' : ''; + // Opt into the UI build, whose verified asset pack is published beside the + // binary, with CBM_VARIANT=ui. Default is the standard (headless) build. + const ui = selectedVariant === 'ui' ? 'ui-' : ''; const archive = `codebase-memory-mcp-${ui}${platform}-${arch}${variant}.${ext}`; const url = `https://github.com/${REPO}/releases/download/v${VERSION}/${archive}`; @@ -596,19 +1069,29 @@ async function main() { await download(url, tmpArchive); await verifyChecksum(tmpArchive, archive); - // Validate the complete archive namespace, then extract only executables. + // Validate the complete archive namespace, then extract only the runtime set. + let runtimeNames; if (ext === 'tar.gz') { - extractExactTarArchive( - tmpArchive, tmpDir, archiveNames, binName, + runtimeNames = extractExactTarArchive( + tmpArchive, tmpDir, archiveNames, extractedNames, + selectedVariant === 'ui', ); } else { - extractZipOnWindows(tmpArchive, tmpDir, archiveNames, extractedNames); + extractZipOnWindows( + tmpArchive, tmpDir, archiveNames, extractedNames, + selectedVariant === 'ui', + ); + const packNames = fs.readdirSync(tmpDir).filter((name) => UI_PACK_PATTERN.test(name)); + runtimeNames = [ + ...extractedNames, + ...(selectedVariant === 'ui' ? packNames : []), + ]; } // Validate extracted paths don't escape tmpDir (tar-slip defense). const resolvedTmpDir = path.resolve(tmpDir); const extractedPaths = new Map(); - for (const name of extractedNames) { + for (const name of runtimeNames) { const extracted = path.join(tmpDir, name); const resolvedExtracted = path.resolve(extracted); if (!resolvedExtracted.startsWith(resolvedTmpDir + path.sep)) { @@ -616,46 +1099,22 @@ async function main() { } const status = fs.lstatSync(extracted, { throwIfNoEntry: false }); if (!status || !status.isFile() || status.isSymbolicLink()) { - throw new Error(`Required binary not found as a regular file: ${extracted}`); + throw new Error(`Required runtime member not found as a regular file: ${extracted}`); } - fs.chmodSync(extracted, 0o755); + fs.chmodSync(extracted, name === binName ? 0o755 : 0o644); extractedPaths.set(name, extracted); } - if (platform === 'windows') { - // The launcher resolves the adjacent portable payload in this directory. - verifyCandidate(extractedPaths.get(WINDOWS_BINARY_NAME)); - } else { - verifyCandidate(extractedPaths.get(binName)); - } - - if (platform === 'windows') { - installWindowsBinaryAtomically(tmpDir, BIN_DIR); - } else { - const staged = path.join( - BIN_DIR, - `.${binName}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`, - ); - try { - fs.copyFileSync(extractedPaths.get(binName), staged, fs.constants.COPYFILE_EXCL); - fs.chmodSync(staged, 0o755); - verifyCandidate(staged); - try { - fs.renameSync(staged, binPath); - } catch (publishError) { - if (!filesEqualSha256(staged, binPath)) throw publishError; - } - verifyCandidate(binPath); - } finally { - try { fs.unlinkSync(staged); } catch (_) { /* renamed or already absent */ } - } - } + verifyCandidate(extractedPaths.get(binName)); + publishRuntimeSetWithRecovery( + tmpDir, cacheDir, binName, selectedVariant, + ); process.stdout.write('codebase-memory-mcp: ready.\n'); if (platform === 'windows') { process.stdout.write( 'Windows package cache is portable. Run "codebase-memory-mcp install --yes" ' + - 'to create the managed launcher (use "npx codebase-memory-mcp install --yes" ' + + 'to create the managed installation (use "npx codebase-memory-mcp install --yes" ' + 'for a local npm install). Package updates/removal remain ' + '"npm install codebase-memory-mcp@latest" and ' + '"npm uninstall codebase-memory-mcp" (add -g for a global install).\n', @@ -675,10 +1134,21 @@ if (require.main === module || module.parent == null) { } module.exports = { + INTEGRATIONS_NAME, UNIX_ARCHIVE_NAMES, + UI_PACK_PATTERN, WINDOWS_BINARY_NAME, + acquireRuntimeLock, + cacheDirForVariant, extractExactTarArchive, - installWindowsBinaryAtomically, + publishRuntimeSetWithRecovery, + releaseRuntimeLock, + refreshRuntimeLock, + runtimeSetNames, + runtimeSetReady, + runtimeSetReadyLocked, + runtimeVariant, + tryReclaimRuntimeLock, validateExactTarMemberListing, - windowsBinaryReady, + validateRegularTarMemberListing, }; diff --git a/pkg/npm/package.json b/pkg/npm/package.json index 0b076b92f2..d82a14bd18 100644 --- a/pkg/npm/package.json +++ b/pkg/npm/package.json @@ -1,7 +1,7 @@ { "name": "codebase-memory-mcp", "version": "0.8.1", - "description": "Fast code intelligence engine for AI coding agents — single static binary MCP server", + "description": "Fast code intelligence for AI coding agents — installs a verified native runtime set", "mcpName": "io.github.DeusData/codebase-memory-mcp", "license": "MIT", "repository": { diff --git a/pkg/npm/test/package-publication.test.js b/pkg/npm/test/package-publication.test.js index 560aec700f..9e5849ff11 100644 --- a/pkg/npm/test/package-publication.test.js +++ b/pkg/npm/test/package-publication.test.js @@ -1,17 +1,27 @@ 'use strict'; const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const { spawn } = require('node:child_process'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const test = require('node:test'); const { + INTEGRATIONS_NAME, UNIX_ARCHIVE_NAMES, WINDOWS_BINARY_NAME, + acquireRuntimeLock, + cacheDirForVariant, extractExactTarArchive, - installWindowsBinaryAtomically, + publishRuntimeSetWithRecovery, + runtimeSetReady, + runtimeSetReadyLocked, + releaseRuntimeLock, + tryReclaimRuntimeLock, validateExactTarMemberListing, + validateRegularTarMemberListing, } = require('../install.js'); function exactUnixListing(extra = []) { @@ -33,28 +43,81 @@ test('Unix archive validation rejects traversal and unexpected members', () => { ); }); -test('Unix extraction requests only the validated root executable', () => { +test('release archives require the integrations sidecar', () => { + assert.ok( + UNIX_ARCHIVE_NAMES.includes(INTEGRATIONS_NAME), + 'the authenticated archive namespace must include the runtime integrations sidecar', + ); +}); + +test('UI Unix archive validation accepts exactly one content-addressed pack', () => { + const pack = `cbm-ui-${'a'.repeat(64)}.pack`; + assert.doesNotThrow(() => validateExactTarMemberListing( + exactUnixListing([pack]), UNIX_ARCHIVE_NAMES, true, + )); + assert.throws( + () => validateExactTarMemberListing( + exactUnixListing([pack, `cbm-ui-${'b'.repeat(64)}.pack`]), + UNIX_ARCHIVE_NAMES, + true, + ), + /exactly one|unexpected or duplicate/, + ); +}); + +test('Unix extraction requests only the validated runtime set', () => { const calls = []; const runner = (command, args) => { calls.push({ command, args: [...args] }); - return calls.length === 1 ? exactUnixListing() : Buffer.alloc(0); + if (args[0] === '-tzf') return exactUnixListing(); + if (args[0] === '-tvzf') { + return UNIX_ARCHIVE_NAMES.map( + (name) => `-rw-r--r-- 0 owner group 1 Jan 1 00:00 ${name}`, + ).join('\n') + '\n'; + } + return Buffer.alloc(0); }; extractExactTarArchive( '/tmp/release.tar.gz', '/tmp/extract', UNIX_ARCHIVE_NAMES, - 'codebase-memory-mcp', runner, + ['codebase-memory-mcp', INTEGRATIONS_NAME], false, runner, ); assert.deepEqual(calls[0].args, ['-tzf', '/tmp/release.tar.gz']); + assert.deepEqual(calls[1].args, ['-tvzf', '/tmp/release.tar.gz']); assert.deepEqual( - calls[1].args, - ['-xzf', '/tmp/release.tar.gz', '-C', '/tmp/extract', 'codebase-memory-mcp'], + calls[2].args, + [ + '-xzf', '/tmp/release.tar.gz', '-C', '/tmp/extract', + 'codebase-memory-mcp', INTEGRATIONS_NAME, + ], ); }); -function writeBinary(directory, tag) { +test('Unix archive validation rejects hardlink members', () => { + const listing = UNIX_ARCHIVE_NAMES.map( + (name, index) => `${index === 0 ? 'h' : '-'}rw-r--r-- ${name}`, + ).join('\n') + '\n'; + assert.throws( + () => validateRegularTarMemberListing(listing, UNIX_ARCHIVE_NAMES.length), + /independent regular files/, + ); +}); + +function writeRuntimeSet(directory, tag, variant = 'standard') { fs.mkdirSync(directory, { recursive: true }); fs.writeFileSync(path.join(directory, WINDOWS_BINARY_NAME), `binary:${tag}`); + fs.writeFileSync(path.join(directory, INTEGRATIONS_NAME), `integrations:${tag}`); + if (variant === 'ui') { + const contents = Buffer.from(`pack:${tag}`); + const digest = crypto.createHash('sha256').update(contents).digest('hex'); + fs.writeFileSync( + path.join(directory, `cbm-ui-${digest}.pack`), + contents, + ); + return `cbm-ui-${digest}.pack`; + } + return null; } function fakeBinaryVerifier(binaryPath) { @@ -81,15 +144,39 @@ function assertBinary(directory, tag) { fs.readFileSync(path.join(directory, WINDOWS_BINARY_NAME), 'utf8'), `binary:${tag}`, ); + assert.equal( + fs.readFileSync(path.join(directory, INTEGRATIONS_NAME), 'utf8'), + `integrations:${tag}`, + ); +} + +async function waitForCrashMarker(marker, child, stderr) { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (fs.existsSync(marker)) return; + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error(`publication helper exited before crash gate: ${stderr()}`); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`publication helper did not reach crash gate: ${stderr()}`); +} + +function waitForChildExit(child) { + return new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + }); } test('Windows publication repairs a corrupt cached binary', () => { withBinaryDirectories(({ source, destination }) => { - writeBinary(source, 'candidate'); - writeBinary(destination, 'old'); + writeRuntimeSet(source, 'candidate'); + writeRuntimeSet(destination, 'old'); fs.writeFileSync(path.join(destination, WINDOWS_BINARY_NAME), 'corrupt'); - installWindowsBinaryAtomically(source, destination, fakeBinaryVerifier); + publishRuntimeSetWithRecovery( + source, destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier, + ); assertBinary(destination, 'candidate'); }); @@ -97,9 +184,11 @@ test('Windows publication repairs a corrupt cached binary', () => { test('Windows publication installs into an empty cache', () => { withBinaryDirectories(({ source, destination }) => { - writeBinary(source, 'candidate'); + writeRuntimeSet(source, 'candidate'); - installWindowsBinaryAtomically(source, destination, fakeBinaryVerifier); + publishRuntimeSetWithRecovery( + source, destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier, + ); assertBinary(destination, 'candidate'); }); @@ -107,11 +196,334 @@ test('Windows publication installs into an empty cache', () => { test('Windows publication preserves a valid concurrent winner', () => { withBinaryDirectories(({ source, destination }) => { - writeBinary(source, 'loser'); - writeBinary(destination, 'winner'); + writeRuntimeSet(source, 'loser'); + writeRuntimeSet(destination, 'winner'); - installWindowsBinaryAtomically(source, destination, fakeBinaryVerifier); + publishRuntimeSetWithRecovery( + source, destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier, + ); assertBinary(destination, 'winner'); }); }); + +test('runtime readiness requires the complete variant-specific set', () => { + withBinaryDirectories(({ destination }) => { + writeRuntimeSet(destination, 'standard'); + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier), + true, + ); + fs.unlinkSync(path.join(destination, INTEGRATIONS_NAME)); + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier), + false, + ); + + writeRuntimeSet(destination, 'ui', 'ui'); + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'ui', fakeBinaryVerifier), + true, + ); + fs.writeFileSync( + path.join(destination, `cbm-ui-${'b'.repeat(64)}.pack`), + 'second pack', + ); + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'ui', fakeBinaryVerifier), + false, + ); + }); +}); + +test('UI readiness rejects and publication repairs a digest-mismatched pack', () => { + withBinaryDirectories(({ source, destination }) => { + const sourcePack = writeRuntimeSet(source, 'candidate', 'ui'); + const corruptPack = writeRuntimeSet(destination, 'old', 'ui'); + fs.writeFileSync(path.join(destination, corruptPack), 'corrupt pack bytes'); + + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'ui', fakeBinaryVerifier), + false, + 'a well-shaped pack filename must not authenticate different bytes', + ); + publishRuntimeSetWithRecovery( + source, destination, WINDOWS_BINARY_NAME, 'ui', fakeBinaryVerifier, + ); + + assertBinary(destination, 'candidate'); + assert.equal(fs.readFileSync(path.join(destination, sourcePack), 'utf8'), 'pack:candidate'); + assert.equal(fs.existsSync(path.join(destination, corruptPack)), false); + }); +}); + +test('standard and UI cache directories do not collide', () => { + assert.notEqual(cacheDirForVariant('standard'), cacheDirForVariant('ui')); +}); + +test('runtime publication commits sidecars before the binary', () => { + withBinaryDirectories(({ source, destination }) => { + writeRuntimeSet(source, 'candidate', 'ui'); + const committed = []; + const renameSync = fs.renameSync; + fs.renameSync = (sourcePath, destinationPath) => { + const name = path.basename(destinationPath); + if (path.dirname(destinationPath) === destination && + (name === WINDOWS_BINARY_NAME || name === INTEGRATIONS_NAME || + /^cbm-ui-[0-9a-f]{64}\.pack$/.test(name))) { + committed.push(name); + } + return renameSync(sourcePath, destinationPath); + }; + try { + publishRuntimeSetWithRecovery( + source, destination, WINDOWS_BINARY_NAME, 'ui', fakeBinaryVerifier, + ); + } finally { + fs.renameSync = renameSync; + } + + assert.equal(committed[0], INTEGRATIONS_NAME); + assert.match(committed[1], /^cbm-ui-[0-9a-f]{64}\.pack$/); + assert.equal(committed[2], WINDOWS_BINARY_NAME); + }); +}); + +test('a killed publisher is reconciled on the next locked readiness check', async () => { + const helperSource = String.raw` + 'use strict'; + const fs = require('node:fs'); + const path = require('node:path'); + const [installPath, source, destination, marker, crashName] = process.argv.slice(1); + const install = require(installPath); + const renameSync = fs.renameSync; + fs.renameSync = (sourcePath, destinationPath) => { + const result = renameSync(sourcePath, destinationPath); + if (destinationPath === path.join(destination, crashName)) { + fs.writeFileSync(marker, 'reached\n'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000); + } + return result; + }; + const verifier = (binaryPath) => { + if (!/^binary:(.+)$/.test(fs.readFileSync(binaryPath, 'utf8'))) { + throw new Error('test binary failed verification'); + } + }; + install.publishRuntimeSetWithRecovery( + source, destination, install.WINDOWS_BINARY_NAME, 'standard', verifier, + ); + `; + + for (const [crashName, expectedReady] of [ + [INTEGRATIONS_NAME, false], + [WINDOWS_BINARY_NAME, true], + ]) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cbm-npm-crash-test-')); + const source = path.join(root, 'source'); + const destination = path.join(root, 'destination'); + const marker = path.join(root, 'crash-reached'); + fs.mkdirSync(destination); + writeRuntimeSet(source, 'candidate'); + writeRuntimeSet(destination, 'old'); + fs.writeFileSync( + path.join(destination, WINDOWS_BINARY_NAME), 'corrupt:old', + ); + let stderr = ''; + const child = spawn(process.execPath, [ + '-e', + helperSource, + path.join(__dirname, '..', 'install.js'), + source, + destination, + marker, + crashName, + ], { stdio: ['ignore', 'ignore', 'pipe'] }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + const exitPromise = waitForChildExit(child); + try { + await waitForCrashMarker(marker, child, () => stderr); + const backups = fs.readdirSync(destination) + .filter((name) => name.startsWith('.cbm-runtime-backup-')); + assert.equal(backups.length, 1, 'killed publisher did not retain one transaction'); + assert.equal( + fs.lstatSync(path.join(destination, backups[0])).isDirectory(), + true, + ); + assert.equal( + fs.existsSync(path.join( + destination, backups[0], '.retirement-complete', + )), + true, + ); + + assert.equal(child.kill('SIGKILL'), true); + const exit = await exitPromise; + assert.ok( + exit.signal !== null || exit.code !== 0, + 'publication helper was not killed', + ); + + const ready = runtimeSetReadyLocked( + destination, + WINDOWS_BINARY_NAME, + 'standard', + fakeBinaryVerifier, + ); + assert.equal(ready, expectedReady); + if (!ready) { + assert.equal( + fs.readFileSync(path.join(destination, WINDOWS_BINARY_NAME), 'utf8'), + 'corrupt:old', + ); + assert.equal( + fs.readFileSync(path.join(destination, INTEGRATIONS_NAME), 'utf8'), + 'integrations:old', + ); + publishRuntimeSetWithRecovery( + source, + destination, + WINDOWS_BINARY_NAME, + 'standard', + fakeBinaryVerifier, + ); + } + assertBinary(destination, 'candidate'); + assert.equal( + fs.readdirSync(destination).some( + (name) => name.startsWith('.cbm-runtime-backup-'), + ), + false, + ); + assert.equal( + fs.existsSync(path.join(destination, '.codebase-memory-mcp-runtime.lock')), + false, + ); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill('SIGKILL'); + await exitPromise; + } + fs.rmSync(root, { recursive: true, force: true }); + } + } +}); + +test('a stalled creator never deletes a successor runtime lock', () => { + withBinaryDirectories(({ destination }) => { + let successor; + let injected = false; + assert.throws( + () => acquireRuntimeLock(destination, () => { + if (injected) return; + injected = true; + successor = acquireRuntimeLock(destination); + throw new Error('injected stalled creator abort'); + }), + /injected stalled creator abort/, + ); + + assert.ok(successor, 'the successor must acquire the reclaimed lock'); + assert.ok( + fs.existsSync(successor.lockPath), + 'the failed creator must not recursively remove its successor', + ); + releaseRuntimeLock(successor); + }); +}); + +test('runtime readiness rejects multiply-linked cache leaves', () => { + withBinaryDirectories(({ destination }) => { + writeRuntimeSet(destination, 'linked'); + fs.linkSync( + path.join(destination, INTEGRATIONS_NAME), + path.join(destination, 'integrations-hardlink'), + ); + assert.equal( + runtimeSetReady(destination, WINDOWS_BINARY_NAME, 'standard', fakeBinaryVerifier), + false, + ); + }); +}); + +test('orphan reconciliation rejects multiply-linked backup members', () => { + withBinaryDirectories(({ destination }) => { + const backup = path.join( + destination, `.cbm-runtime-backup-${'a'.repeat(32)}`, + ); + fs.mkdirSync(backup); + fs.writeFileSync(path.join(backup, '.retirement-complete'), ''); + const member = path.join(backup, WINDOWS_BINARY_NAME); + fs.writeFileSync(member, 'binary:old'); + fs.linkSync(member, path.join(destination, 'backup-hardlink-copy')); + + assert.throws( + () => runtimeSetReadyLocked( + destination, + WINDOWS_BINARY_NAME, + 'standard', + fakeBinaryVerifier, + ), + /unsafe package-cache backup member/, + ); + assert.equal(fs.existsSync(backup), true, 'unsafe backup was mutated'); + assert.equal( + fs.existsSync(path.join(destination, '.codebase-memory-mcp-runtime.lock')), + false, + ); + }); +}); + +test('an expired lease never reclaims a live owner', () => { + withBinaryDirectories(({ destination }) => { + const lockPath = path.join(destination, '.codebase-memory-mcp-runtime.lock'); + const owner = { + pid: process.pid, + token: 'a'.repeat(32), + lease_expires_ms: Date.now() - 1, + }; + const ownerRecord = `${JSON.stringify(owner)}\n`; + fs.writeFileSync(lockPath, ownerRecord); + const before = fs.lstatSync(lockPath, { bigint: true }); + + assert.equal(tryReclaimRuntimeLock(lockPath, 'b'.repeat(32)), false); + const after = fs.lstatSync(lockPath, { bigint: true }); + assert.equal(fs.readFileSync(lockPath, 'utf8'), ownerRecord); + assert.equal(JSON.parse(fs.readFileSync(lockPath, 'utf8')).token, owner.token); + assert.equal(after.dev, before.dev); + assert.equal(after.ino, before.ino); + }); +}); + +test('a stale legacy lock never reclaims a live owner', () => { + withBinaryDirectories(({ destination }) => { + const lockPath = path.join(destination, '.codebase-memory-mcp-runtime.lock'); + const ownerRecord = `${JSON.stringify({ + pid: process.pid, + token: 'a'.repeat(32), + })}\n`; + fs.mkdirSync(lockPath); + fs.writeFileSync(path.join(lockPath, 'owner.json'), ownerRecord); + const stale = new Date(Date.now() - 3_600_001); + fs.utimesSync(lockPath, stale, stale); + const before = fs.lstatSync(lockPath, { bigint: true }); + + assert.equal(tryReclaimRuntimeLock(lockPath, 'b'.repeat(32)), false); + const after = fs.lstatSync(lockPath, { bigint: true }); + assert.equal(fs.readFileSync(path.join(lockPath, 'owner.json'), 'utf8'), ownerRecord); + assert.equal(after.dev, before.dev); + assert.equal(after.ino, before.ino); + }); +}); + +test('a stale ownerless lock is reclaimed', () => { + withBinaryDirectories(({ destination }) => { + const lockPath = path.join(destination, '.codebase-memory-mcp-runtime.lock'); + fs.writeFileSync(lockPath, ''); + const stale = new Date(Date.now() - 30_001); + fs.utimesSync(lockPath, stale, stale); + + assert.equal(tryReclaimRuntimeLock(lockPath, 'b'.repeat(32)), true); + assert.equal(fs.existsSync(lockPath), false); + }); +}); diff --git a/pkg/npm/test/windows-launcher.test.js b/pkg/npm/test/windows-launcher.test.js index 756b238cb3..da19f459b6 100644 --- a/pkg/npm/test/windows-launcher.test.js +++ b/pkg/npm/test/windows-launcher.test.js @@ -1,11 +1,15 @@ 'use strict'; const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); const fs = require('node:fs'); const path = require('node:path'); const test = require('node:test'); const vm = require('node:vm'); +const UI_PACK_BYTES = Buffer.from('authenticated UI pack'); +const UI_PACK_NAME = `cbm-ui-${crypto.createHash('sha256').update(UI_PACK_BYTES).digest('hex')}.pack`; + class ExitSignal extends Error { constructor(code) { super(`process.exit(${code})`); @@ -13,26 +17,77 @@ class ExitSignal extends Error { } } -function runShim(targetPlatform, arguments_, childStatus) { +function runShim(targetPlatform, arguments_, childStatus, options = {}) { const shimPath = path.join(__dirname, '..', 'bin.js'); const source = fs.readFileSync(shimPath, 'utf8').replace(/^#![^\n]*\n/, ''); const calls = []; + const lockEvents = []; let stderr = ''; + let installed = false; const fakeProcess = { platform: targetPlatform, + env: { + CBM_VARIANT: options.variant || '', + LOCALAPPDATA: 'C:\\Users\\test\\AppData\\Local', + }, argv: ['node.exe', shimPath, ...arguments_], execPath: 'node.exe', stderr: { write: (text) => { stderr += String(text); } }, exit: (code) => { throw new ExitSignal(code); }, }; const fakeFs = { - existsSync: () => true, + lstatSync: (candidate) => { + if (options.missingSidecar && !installed && + path.basename(candidate) === 'cbm-integrations.json') { + const error = new Error('missing sidecar'); + error.code = 'ENOENT'; + throw error; + } + return { isFile: () => true, isSymbolicLink: () => false, nlink: 1 }; + }, + readFileSync: () => options.corruptPack && !installed + ? Buffer.from('corrupt UI pack') + : UI_PACK_BYTES, + readdirSync: () => options.variant === 'ui' + ? [UI_PACK_NAME] + : [], }; const fakeChildProcess = { + spawn: (executable, args, options) => { + calls.push({ executable, args, options }); + return { + on: (event, callback) => { + if (event === 'close') callback(childStatus); + return this; + }, + }; + }, spawnSync: (executable, args, options) => { calls.push({ executable, args, options }); - return { status: calls.length === 1 ? 0 : childStatus }; + if (executable === fakeProcess.execPath) { + installed = true; + return { status: 0 }; + } + if (args[0] === '--verify-runtime-assets') return { status: 0 }; + return { status: childStatus }; + }, + }; + const fakeLifecycle = { + acquireRuntimeLock: () => { + lockEvents.push('acquire'); + return { lockPath: 'lock', token: 'token', fd: 1 }; + }, + refreshRuntimeLock: () => { lockEvents.push('refresh'); }, + releaseRuntimeLock: () => { lockEvents.push('release'); }, + runtimeSetReady: () => true, + runtimeSetReadyLocked: (directory, binaryName) => { + const structurallyReady = installed || (!options.missingSidecar && !options.corruptPack); + if (!structurallyReady) return false; + const probe = fakeChildProcess.spawnSync( + path.join(directory, binaryName), ['--verify-runtime-assets'], { stdio: 'ignore' }, + ); + return probe.status === 0; }, }; const sandbox = { @@ -47,8 +102,11 @@ function runShim(targetPlatform, arguments_, childStatus) { require: (specifier) => { if (specifier === 'fs') return fakeFs; if (specifier === 'child_process') return fakeChildProcess; + if (specifier === './install.js') return fakeLifecycle; return require(specifier); }, + clearInterval: () => {}, + setInterval: () => 1, setTimeout, }; @@ -59,10 +117,10 @@ function runShim(targetPlatform, arguments_, childStatus) { if (!(error instanceof ExitSignal)) throw error; exitCode = error.code; } - return { calls, exitCode, stderr }; + return { calls, exitCode, lockEvents, stderr }; } -test('Windows npm shim probes and executes the cached launcher', () => { +test('Windows npm shim authenticates assets and executes the cached binary', () => { const observed = runShim('win32', ['--version'], 0); assert.equal(observed.exitCode, 0); @@ -74,25 +132,82 @@ test('Windows npm shim probes and executes the cached launcher', () => { 'codebase-memory-mcp.payload.exe', ); } + assert.deepEqual(Array.from(observed.calls[0].args), ['--verify-runtime-assets']); assert.deepEqual(Array.from(observed.calls[1].args), ['--version']); assert.equal(observed.calls[1].options.stdio, 'inherit'); }); -test('Windows npm shim keeps package-manager guidance after launcher refusal', () => { +test('npm update is routed to the package manager without invoking absent install.sh', () => { const observed = runShim('win32', ['update'], 1); - assert.equal(observed.exitCode, 1); - assert.equal(path.basename(observed.calls[1].executable), 'codebase-memory-mcp.exe'); + assert.equal(observed.exitCode, 2); + assert.equal(observed.calls.length, 0, 'update guidance must not launch cached code'); assert.match(observed.stderr, /npm install codebase-memory-mcp@latest/); - assert.match(observed.stderr, /install --yes/); + assert.match(observed.stderr, /codebase-memory-mcp install --yes/); + assert.doesNotMatch(observed.stderr, /install\.sh/); }); -test('non-Windows npm shim keeps its native payload execution path', () => { +test('nested update token is forwarded instead of treated as package update', () => { + const observed = runShim('win32', ['daemon', 'update'], 0); + + assert.equal(observed.exitCode, 0); + assert.equal(observed.calls.length, 2); + assert.deepEqual(Array.from(observed.calls[1].args), ['daemon', 'update']); + assert.doesNotMatch(observed.stderr, /npm install codebase-memory-mcp@latest/); +}); + +test('non-Windows npm shim executes its cached binary directly', () => { const observed = runShim('darwin', ['--version'], 0); assert.equal(observed.exitCode, 0); - assert.equal(observed.calls.length, 1); - assert.equal(path.basename(observed.calls[0].executable), 'codebase-memory-mcp'); + assert.equal(observed.calls.length, 2); + assert.deepEqual(Array.from(observed.calls[0].args), ['--verify-runtime-assets']); + assert.equal(path.basename(observed.calls[1].executable), 'codebase-memory-mcp'); +}); + +test('npm shim selects a separate UI runtime set', () => { + const observed = runShim('darwin', ['--version'], 0, { variant: 'ui' }); + + assert.equal(observed.exitCode, 0); + assert.equal(path.basename(path.dirname(observed.calls[1].executable)), 'ui'); +}); + +test('npm shim repairs a cache whose integrations sidecar is missing', () => { + const observed = runShim( + 'darwin', ['--version'], 0, { missingSidecar: true }, + ); + + assert.equal(observed.exitCode, 0); + assert.equal(observed.calls[0].executable, 'node.exe'); + assert.equal(path.basename(observed.calls[0].args[0]), 'install.js'); + assert.deepEqual(Array.from(observed.calls[1].args), ['--verify-runtime-assets']); + assert.equal(path.basename(observed.calls[2].executable), 'codebase-memory-mcp'); +}); + +test('npm shim repairs a UI cache whose pack digest does not match its name', () => { + const observed = runShim( + 'darwin', ['--version'], 0, { variant: 'ui', corruptPack: true }, + ); + + assert.equal(observed.exitCode, 0); + assert.equal(observed.calls[0].executable, 'node.exe'); + assert.equal(path.basename(observed.calls[0].args[0]), 'install.js'); + assert.deepEqual(Array.from(observed.calls[1].args), ['--verify-runtime-assets']); + assert.equal(path.basename(observed.calls[2].executable), 'codebase-memory-mcp'); +}); + +test('wrapper uninstall locks the cache and targets the managed install by default', () => { + const observed = runShim('win32', ['uninstall', '--yes'], 0); + + assert.equal(observed.exitCode, 0); + assert.deepEqual(observed.lockEvents, ['acquire', 'refresh', 'release']); + assert.deepEqual( + Array.from(observed.calls[1].args), + [ + 'uninstall', '--yes', '--dir', + 'C:\\Users\\test\\AppData\\Local/Programs/codebase-memory-mcp', + ], + ); }); test('PowerShell install mutation runs through the downloaded binary', () => { @@ -111,8 +226,12 @@ test('PowerShell install mutation runs through the downloaded binary', () => { ); // One binary ships per platform — no launcher/payload pair to resolve. assert.doesNotMatch(installer, /payload/i); - // A running .exe cannot be overwritten, so an in-place update has to retire - // the existing binary by renaming it aside first. This script IS the Windows - // update path, so losing that step would silently break every update. - assert.match(installer, /Move-Item[\s\S]{0,80}\$Dest[\s\S]{0,40}\$retired/); + // The candidate owns activation and rollback under the native guard. The + // script must not mutate the installed binary before handing it the request. + const handoff = installer.indexOf('& $DownloadedBinary @InstallArgs'); + assert.ok(handoff > 0, 'downloaded candidate handoff is missing'); + assert.doesNotMatch( + installer.slice(0, handoff), + /(?:Move-Item|Copy-Item|Remove-Item)[^\n]*(?:\$Dest|\$InstallDir[^\n]*\$BinName)/, + ); }); diff --git a/pkg/pypi/README.md b/pkg/pypi/README.md index 97d09ccaf5..f75af87159 100644 --- a/pkg/pypi/README.md +++ b/pkg/pypi/README.md @@ -4,7 +4,7 @@ mcp-name: io.github.DeusData/codebase-memory-mcp **Fast code intelligence engine for AI coding agents.** Indexes an average repository in milliseconds, the Linux kernel (28M LOC) in 3 minutes. Answers structural queries in under 1ms. -This package installs the `codebase-memory-mcp` binary from [GitHub Releases](https://github.com/DeusData/codebase-memory-mcp/releases). The binary is downloaded on first run and cached in your OS cache directory. +This Python wrapper downloads the selected `codebase-memory-mcp` runtime set from [GitHub Releases](https://github.com/DeusData/codebase-memory-mcp/releases) on first run and verifies it before publishing it in your OS cache directory. The standard set contains the native executable and authenticated integration asset; `CBM_VARIANT=ui` additionally selects the content-addressed UI pack. ## Installation @@ -14,6 +14,8 @@ pip install codebase-memory-mcp pipx install codebase-memory-mcp ``` +To use the UI variant, set `CBM_VARIANT=ui` when invoking the wrapper (and consistently for any package-managed update or reinstall). + ## Usage ```bash @@ -27,7 +29,7 @@ codebase-memory-mcp --help |---------|-------------| | macOS | arm64, amd64 | | Linux | arm64, amd64 | -| Windows | amd64 | +| Windows | arm64, amd64 | ## Full documentation diff --git a/pkg/pypi/pyproject.toml b/pkg/pypi/pyproject.toml index 9a430644a0..18807810db 100644 --- a/pkg/pypi/pyproject.toml +++ b/pkg/pypi/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "codebase-memory-mcp" version = "0.8.1" -description = "Fast code intelligence engine for AI coding agents — single static binary MCP server" +description = "Fast code intelligence for AI coding agents — installs a verified native runtime set" readme = "README.md" license = { text = "MIT" } requires-python = ">=3.8" diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index 473c2d026e..05f854e6f4 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -1,13 +1,18 @@ """Downloads codebase-memory-mcp on first run, then runs its native entry point.""" +import errno import hashlib +import json import os import platform +import re +import secrets import shutil import stat import subprocess import sys import tempfile +import threading import time import urllib.error import urllib.parse @@ -16,8 +21,18 @@ REPO = "DeusData/codebase-memory-mcp" _WINDOWS_BINARY_NAME = "codebase-memory-mcp.exe" +_INTEGRATIONS_NAME = "cbm-integrations.json" +_UI_PACK_RE = re.compile(r"cbm-ui-[0-9a-f]{64}\.pack\Z") +_UNIX_ARCHIVE_NAMES = ( + "codebase-memory-mcp", + _INTEGRATIONS_NAME, + "LICENSE", + "install.sh", + "THIRD_PARTY_NOTICES.md", +) _WINDOWS_ARCHIVE_NAMES = ( _WINDOWS_BINARY_NAME, + _INTEGRATIONS_NAME, "LICENSE", "install.ps1", "THIRD_PARTY_NOTICES.md", @@ -32,6 +47,18 @@ _CANDIDATE_TIMEOUT_SECONDS = 15 _MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024 _REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) +_RUNTIME_LOCK_NAME = ".codebase-memory-mcp-runtime.lock" +_RUNTIME_LOCK_WAIT_SECONDS = 45 +_RUNTIME_OWNERLESS_STALE_SECONDS = 30 +_RUNTIME_LOCK_POLL_SECONDS = 0.025 +_RUNTIME_LOCK_LEASE_SECONDS = 300 +_RUNTIME_LOCK_HEARTBEAT_SECONDS = 20 +_RUNTIME_LEGACY_STALE_SECONDS = 3600 +_RUNTIME_BACKUP_PREFIX = ".cbm-runtime-backup-" +_RUNTIME_BACKUP_RE = re.compile(r"\.cbm-runtime-backup-[0-9a-f]{32}\Z") +_RUNTIME_BACKUP_RETIRED_MARKER = ".retirement-complete" +_RUNTIME_BACKUP_CLEANUP_MARKER = ".cleanup-only" +_runtime_lock_claim_observer = None class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -111,7 +138,7 @@ def _verify_candidate(path: Path) -> None: """Require a staged native candidate to execute successfully.""" try: subprocess.run( - [str(path), "--version"], + [str(path), "--verify-runtime-assets"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -133,8 +160,8 @@ def _file_sha256(path: Path) -> str: def _files_equal_sha256(left: Path, right: Path) -> bool: try: return ( - left.is_file() - and right.is_file() + _regular_file(left) + and _regular_file(right) and left.stat().st_size == right.stat().st_size and _file_sha256(left) == _file_sha256(right) ) @@ -142,34 +169,77 @@ def _files_equal_sha256(left: Path, right: Path) -> bool: return False -def _safe_extract_tar(tf, dest: str) -> None: - """Extract a tarfile to dest, rejecting path-traversal entries. - - Uses the tarfile data filter on Python >=3.12 (PEP 706), falls back to - manual per-member path validation on older Pythons. Mitigates the - classic tar-slip / Zip Slip vulnerability (CWE-22). - """ - if hasattr(tf, "extraction_filter") or sys.version_info >= (3, 12): - tf.extractall(dest, filter="data") - return - - dest_abs = os.path.abspath(dest) - for member in tf.getmembers(): - if member.issym() or member.islnk(): +def _validate_archive_names(names, archive_names, ui: bool, casefold: bool = False): + """Require the exact release root namespace and return its UI pack name.""" + required = tuple(archive_names) + required_set = set(required) + found = set() + seen = set() + ui_pack = None + for name in names: + key = name.casefold() if casefold else name + if key in seen: sys.exit( - f"codebase-memory-mcp: refusing unsafe tar entry " - f"(link: {member.name!r})" + f"codebase-memory-mcp: refusing unsafe archive entry " + f"(duplicate or case conflict: {name!r})" ) - member_abs = os.path.abspath(os.path.join(dest_abs, member.name)) - if not (member_abs == dest_abs or member_abs.startswith(dest_abs + os.sep)): + seen.add(key) + is_ui_pack = ui and _UI_PACK_RE.fullmatch(name) is not None + if name not in required_set and not is_ui_pack: + sys.exit( + f"codebase-memory-mcp: archive must contain only the exact " + f"root files: {', '.join(required)}" + ) + if is_ui_pack: + if ui_pack is not None: + sys.exit( + "codebase-memory-mcp: archive must contain exactly one " + "content-addressed UI pack" + ) + ui_pack = name + else: + found.add(name) + expected_count = len(required) + (1 if ui else 0) + if found != required_set or len(seen) != expected_count or (ui and ui_pack is None): + sys.exit( + f"codebase-memory-mcp: archive must contain exactly one of each " + f"required root file: {', '.join(required)}" + ) + return ui_pack + + +def _safe_extract_tar( + tf, dest: str, archive_names=(), extract_names=(), ui: bool = False +): + """Validate a tar namespace, then extract only its authenticated runtime set.""" + members = tf.getmembers() + for member in members: + if not member.isfile(): sys.exit( f"codebase-memory-mcp: refusing unsafe tar entry " - f"(escapes dest: {member.name!r})" + f"(not a regular root file: {member.name!r})" ) - tf.extractall(dest) + ui_pack = _validate_archive_names( + [member.name for member in members], archive_names, ui + ) + runtime_names = list(extract_names) + if ui_pack is not None: + runtime_names.append(ui_pack) + by_name = {member.name: member for member in members} + dest_abs = os.path.abspath(dest) + for name in runtime_names: + source = tf.extractfile(by_name[name]) + if source is None: + sys.exit(f"codebase-memory-mcp: could not read archive member: {name}") + target = os.path.join(dest_abs, name) + with source, open(target, "xb") as output: + shutil.copyfileobj(source, output) + return tuple(runtime_names) -def _safe_extract_zip(zf, dest: str, archive_names=(), extract_names=()) -> None: +def _safe_extract_zip( + zf, dest: str, archive_names=(), extract_names=(), ui: bool = False +): """Extract a zipfile after validating its Windows-style namespace. Windows treats paths case-insensitively. Rejecting duplicate/case-conflicting @@ -177,10 +247,7 @@ def _safe_extract_zip(zf, dest: str, archive_names=(), extract_names=()) -> None that a portable package-manager shim eventually executes. """ dest_abs = os.path.abspath(dest) - seen = set() - required = tuple(archive_names) - required_set = set(required) - found = set() + names = [] for info in zf.infolist(): raw_name = info.filename name = raw_name.replace("\\", "/") @@ -197,38 +264,31 @@ def _safe_extract_zip(zf, dest: str, archive_names=(), extract_names=()) -> None f"codebase-memory-mcp: refusing unsafe zip entry " f"(invalid path: {raw_name!r})" ) - folded = segments_name.casefold() - if folded in seen: + unix_type = stat.S_IFMT((info.external_attr >> 16) & 0xFFFF) + if info.is_dir() or unix_type not in (0, stat.S_IFREG): sys.exit( f"codebase-memory-mcp: refusing unsafe zip entry " - f"(duplicate or case conflict: {raw_name!r})" + f"(not a regular root file: {raw_name!r})" ) - seen.add(folded) - if name not in required_set or info.is_dir(): - sys.exit( - f"codebase-memory-mcp: archive must contain only the exact " - f"root files: {', '.join(required)}" - ) - found.add(name) + names.append(name) member_abs = os.path.abspath(os.path.join(dest_abs, *segments)) if not (member_abs == dest_abs or member_abs.startswith(dest_abs + os.sep)): sys.exit( f"codebase-memory-mcp: refusing unsafe zip entry " f"(escapes dest: {raw_name!r})" ) - missing = required_set - found - if missing or len(seen) != len(required): - sys.exit( - f"codebase-memory-mcp: archive must contain exactly one of each " - f"required root file: {', '.join(required)}" - ) + ui_pack = _validate_archive_names(names, archive_names, ui, casefold=True) - # Extract only the two validated executable files. This avoids relying on + # Extract only the validated runtime files. This avoids relying on # platform-specific zip path rewriting and always creates regular files. - for name in extract_names: + runtime_names = list(extract_names) + if ui_pack is not None: + runtime_names.append(ui_pack) + for name in runtime_names: target = os.path.join(dest_abs, name) with zf.open(name) as source, open(target, "xb") as output: shutil.copyfileobj(source, output) + return tuple(runtime_names) def _verify_checksum(archive_path: str, archive_name: str, version: str) -> None: @@ -329,14 +389,22 @@ def _cache_dir() -> Path: return base / "codebase-memory-mcp" -def _bin_path(version: str) -> Path: +def _variant() -> str: + return "ui" if os.environ.get("CBM_VARIANT", "").lower() == "ui" else "standard" + + +def _runtime_dir(version: str, variant: str = None) -> Path: + return _cache_dir() / version / (variant or _variant()) + + +def _bin_path(version: str, variant: str = None) -> Path: # One binary per platform: the cached file is the executed file. name = ( _WINDOWS_BINARY_NAME if sys.platform == "win32" else "codebase-memory-mcp" ) - return _cache_dir() / version / name + return _runtime_dir(version, variant) / name def _execution_path(binary: Path, target_platform: str) -> Path: @@ -344,33 +412,777 @@ def _execution_path(binary: Path, target_platform: str) -> Path: return binary -def _windows_binary_ready(version: str) -> bool: - binary = _cache_dir() / version / _WINDOWS_BINARY_NAME - if not binary.is_file(): +def _regular_file(path: Path) -> bool: + try: + status = path.lstat() + return ( + stat.S_ISREG(status.st_mode) + and not stat.S_ISLNK(status.st_mode) + and status.st_nlink == 1 + ) + except OSError: + return False + + +def _ui_pack_path_matches_name(path: Path, name: str) -> bool: + try: + expected = name[len("cbm-ui-") : -len(".pack")] + return _file_sha256(path) == expected + except OSError: return False + + +def _ui_pack_matches_digest(path: Path) -> bool: + return _ui_pack_path_matches_name(path, path.name) + + +def _runtime_set_names(directory: Path, binary_name: str, variant: str): + if not _regular_file(directory / binary_name) or not _regular_file( + directory / _INTEGRATIONS_NAME + ): + return None + try: + pack_like = [ + path + for path in directory.iterdir() + if path.name.startswith("cbm-ui-") and path.name.endswith(".pack") + ] + except OSError: + return None + valid_packs = [ + path + for path in pack_like + if _UI_PACK_RE.fullmatch(path.name) is not None + and _regular_file(path) + and _ui_pack_matches_digest(path) + ] + if variant == "ui": + if len(pack_like) != 1 or len(valid_packs) != 1: + return None + elif pack_like: + return None + return ( + _INTEGRATIONS_NAME, + *((valid_packs[0].name,) if variant == "ui" else ()), + binary_name, + ) + + +def _runtime_set_ready(version: str, verifier=None) -> bool: + variant = _variant() + binary = _bin_path(version, variant) + if _runtime_set_names(binary.parent, binary.name, variant) is None: + return False + if verifier is not None: + try: + verifier(binary) + except RuntimeError: + return False + return True + + +def _require_safe_runtime_directory(directory: Path): + status = directory.lstat() + if not stat.S_ISDIR(status.st_mode) or stat.S_ISLNK(status.st_mode): + raise RuntimeError(f"refusing unsafe package-cache directory: {directory}") + + +def _runtime_set_ready_locked(version: str) -> bool: + binary = _bin_path(version) + binary.parent.mkdir(parents=True, exist_ok=True) + _require_safe_runtime_directory(binary.parent) + lock, _ = _acquire_runtime_lock(binary.parent) + try: + _refresh_runtime_lock(lock) + _reconcile_runtime_backups( + binary.parent, binary.name, _variant(), _verify_candidate, lock + ) + return _runtime_set_ready(version, _verify_candidate) + finally: + _release_runtime_lock(lock) + + +def _runtime_set_fingerprint(directory: Path, binary_name: str, variant: str): + """Identify one complete set so a lock waiter can preserve its winner.""" + names = _runtime_set_names(directory, binary_name, variant) + if names is None: + return None + try: + return tuple( + (name, _file_sha256(directory / name)) for name in names + ) + except OSError: + return None + + +def _staged_runtime_names(staged_paths, binary_name: str, variant: str): + names = set(staged_paths) + required = {binary_name, _INTEGRATIONS_NAME} + pack_names = [ + name + for name in names + if name.startswith("cbm-ui-") and name.endswith(".pack") + ] + if variant == "ui": + if len(pack_names) != 1 or _UI_PACK_RE.fullmatch(pack_names[0]) is None: + raise RuntimeError("staged UI runtime set must contain exactly one pack") + pack_name = pack_names[0] + if not _regular_file(staged_paths[pack_name]) or not _ui_pack_path_matches_name( + staged_paths[pack_name], pack_name + ): + raise RuntimeError("staged UI pack digest does not match its filename") + required.add(pack_name) + runtime_names = (_INTEGRATIONS_NAME, pack_name, binary_name) + else: + if pack_names: + raise RuntimeError("standard runtime set must not contain a UI pack") + runtime_names = (_INTEGRATIONS_NAME, binary_name) + if names != required or any( + not _regular_file(staged_paths[name]) for name in required + ): + raise RuntimeError("staged package cache is incomplete") + return runtime_names + + +def _process_is_alive(pid: int) -> bool: + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + return False + try: + os.kill(pid, 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as exc: + return exc.errno != errno.ESRCH + + +def _runtime_lock_status(lock_path: Path): + try: + return lock_path.lstat() + except OSError: + return None + + +def _same_runtime_lock_object(left, right): + return ( + left is not None + and right is not None + and left.st_dev == right.st_dev + and left.st_ino == right.st_ino + and stat.S_IFMT(left.st_mode) == stat.S_IFMT(right.st_mode) + ) + + +def _read_runtime_lock_owner(lock_path: Path): + try: + status = lock_path.lstat() + if stat.S_ISLNK(status.st_mode) or not ( + stat.S_ISREG(status.st_mode) or stat.S_ISDIR(status.st_mode) + ): + return None + owner_path = lock_path / "owner.json" if stat.S_ISDIR(status.st_mode) else lock_path + owner = json.loads(owner_path.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + return None + pid = owner.get("pid") + token = owner.get("token") + if ( + not isinstance(pid, int) + or isinstance(pid, bool) + or pid <= 0 + or not isinstance(token, str) + or re.fullmatch(r"[0-9a-f]{32}", token) is None + ): + return None + lease_expires = owner.get("lease_expires_ms") + if not isinstance(lease_expires, (int, float)) or isinstance( + lease_expires, bool + ): + lease_expires = None + return pid, token, lease_expires + + +def _restore_displaced_runtime_lock(reclaimed: Path, lock_path: Path, status): + if status is None or not stat.S_ISREG(status.st_mode): + return + try: + os.link(reclaimed, lock_path) + reclaimed.unlink() + except OSError: + # A newer owner holds the canonical name. Preserve the displaced lock; + # never delete a lock whose identity was not the one inspected. + pass + + +def _try_reclaim_runtime_lock(lock_path: Path, contender_token: str): + """Return whether to retry now and whether a contender released the lock.""" + observed_status = _runtime_lock_status(lock_path) + if observed_status is None: + return True, True + if stat.S_ISLNK(observed_status.st_mode) or not ( + stat.S_ISREG(observed_status.st_mode) + or stat.S_ISDIR(observed_status.st_mode) + ): + return False, False + owner = _read_runtime_lock_owner(lock_path) + reclaim = owner is not None and not _process_is_alive(owner[0]) + if owner is None: + reclaim = ( + time.time() - observed_status.st_mtime + >= _RUNTIME_OWNERLESS_STALE_SECONDS + ) + if not reclaim: + return False, False + reclaimed = lock_path.with_name( + f"{lock_path.name}.reclaimed-{contender_token}" + ) + try: + os.rename(lock_path, reclaimed) + except FileNotFoundError: + return True, True + except OSError: + return False, False + moved_status = _runtime_lock_status(reclaimed) + if not _same_runtime_lock_object(observed_status, moved_status): + _restore_displaced_runtime_lock(reclaimed, lock_path, moved_status) + return False, False + moved_owner = _read_runtime_lock_owner(reclaimed) + still_reclaimable = ( + time.time() - moved_status.st_mtime >= _RUNTIME_OWNERLESS_STALE_SECONDS + if moved_owner is None + else not _process_is_alive(moved_owner[0]) + ) + if not still_reclaimable: + _restore_displaced_runtime_lock(reclaimed, lock_path, moved_status) + return False, False + if stat.S_ISDIR(moved_status.st_mode): + shutil.rmtree(reclaimed) + else: + reclaimed.unlink() + return True, False + + +def _write_runtime_lock_owner(owner_fd: int, token: str): + record = ( + json.dumps( + { + "pid": os.getpid(), + "token": token, + "lease_expires_ms": int( + (time.time() + _RUNTIME_LOCK_LEASE_SECONDS) * 1000 + ), + }, + separators=(",", ":"), + ) + + "\n" + ).encode() + os.ftruncate(owner_fd, 0) + os.lseek(owner_fd, 0, os.SEEK_SET) + offset = 0 + while offset < len(record): + offset += os.write(owner_fd, record[offset:]) + os.fsync(owner_fd) + + +def _assert_runtime_lock_owner(lock): + lock_path, token, owner_fd = lock + descriptor_status = os.fstat(owner_fd) + canonical_status = _runtime_lock_status(lock_path) + owner = _read_runtime_lock_owner(lock_path) + if ( + not _same_runtime_lock_object(descriptor_status, canonical_status) + or owner is None + or owner[:2] != (os.getpid(), token) + ): + raise RuntimeError("package-cache publication lock ownership changed") + + +def _refresh_runtime_lock(lock): + _assert_runtime_lock_owner(lock) + _write_runtime_lock_owner(lock[2], lock[1]) + _assert_runtime_lock_owner(lock) + + +def _acquire_runtime_lock(directory: Path): + token = secrets.token_hex(16) + lock_path = directory / _RUNTIME_LOCK_NAME + claim_path = lock_path.with_name(f"{lock_path.name}.claim-{token}") + deadline = time.monotonic() + _RUNTIME_LOCK_WAIT_SECONDS + contended = False + while True: + owner_fd = None + try: + owner_fd = os.open( + claim_path, + os.O_RDWR | os.O_CREAT | os.O_EXCL, + 0o600, + ) + try: + _write_runtime_lock_owner(owner_fd, token) + if _runtime_lock_claim_observer is not None: + _runtime_lock_claim_observer() + os.link(claim_path, lock_path) + claim_path.unlink() + if not _same_runtime_lock_object( + os.fstat(owner_fd), _runtime_lock_status(lock_path) + ): + raise RuntimeError( + "package-cache publication lock ownership changed" + ) + except BaseException: + os.close(owner_fd) + owner_fd = None + try: + claim_path.unlink() + except OSError: + pass + raise + return (lock_path, token, owner_fd), contended + except FileExistsError: + retry, released_by_contender = _try_reclaim_runtime_lock( + lock_path, token + ) + contended = contended or released_by_contender + if retry: + continue + contended = True + if time.monotonic() >= deadline: + raise RuntimeError( + "timed out waiting for package-cache publication lock" + ) + time.sleep(_RUNTIME_LOCK_POLL_SECONDS) + + +def _release_runtime_lock(lock): + lock_path, token, owner_fd = lock + try: + _assert_runtime_lock_owner(lock) + except Exception: + os.close(owner_fd) + raise + released = lock_path.with_name(f"{lock_path.name}.released-{token}") + try: + os.rename(lock_path, released) + except Exception: + os.close(owner_fd) + raise + descriptor_status = os.fstat(owner_fd) + released_status = _runtime_lock_status(released) + owner = _read_runtime_lock_owner(released) + if ( + not _same_runtime_lock_object(descriptor_status, released_status) + or owner is None + or owner[:2] != (os.getpid(), token) + ): + _restore_displaced_runtime_lock(released, lock_path, released_status) + os.close(owner_fd) + raise RuntimeError("package-cache publication lock ownership changed") + try: + released.unlink() + finally: + os.close(owner_fd) + + +# A grouped backup is a small crash journal. The retired marker separates a +# partial retirement from partial publication; cleanup-only makes deletion +# retryable after the destination has been accepted or restored. +def _runtime_backup_target_name(name: str, binary_name: str) -> bool: + return ( + name == binary_name + or name == _INTEGRATIONS_NAME + or (name.startswith("cbm-ui-") and name.endswith(".pack")) + ) + + +def _create_runtime_backup_directory(directory: Path) -> Path: + for _ in range(128): + backup = directory / f"{_RUNTIME_BACKUP_PREFIX}{secrets.token_hex(16)}" + try: + backup.mkdir(mode=0o700) + return backup + except FileExistsError: + continue + raise RuntimeError("could not reserve a package-cache backup transaction") + + +def _create_runtime_backup_marker(backup: Path, name: str): + descriptor = os.open(backup / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: - _verify_candidate(binary) - except RuntimeError: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _inspect_runtime_backup(directory: Path, entry_name: str, binary_name: str): + backup_path = directory / entry_name + try: + directory_status = backup_path.lstat() + except OSError as exc: + raise RuntimeError( + f"refusing unsafe package-cache backup: {backup_path}" + ) from exc + if not stat.S_ISDIR(directory_status.st_mode) or stat.S_ISLNK( + directory_status.st_mode + ): + raise RuntimeError(f"refusing unsafe package-cache backup: {backup_path}") + backup = { + "path": backup_path, + "retired": False, + "cleanup_only": False, + "files": {}, + } + for candidate in sorted(backup_path.iterdir(), key=lambda path: path.name): + status = candidate.lstat() + if candidate.name in ( + _RUNTIME_BACKUP_RETIRED_MARKER, + _RUNTIME_BACKUP_CLEANUP_MARKER, + ): + if ( + not stat.S_ISREG(status.st_mode) + or stat.S_ISLNK(status.st_mode) + or status.st_nlink != 1 + or status.st_size != 0 + ): + raise RuntimeError( + f"refusing unsafe package-cache backup marker: {candidate}" + ) + if candidate.name == _RUNTIME_BACKUP_RETIRED_MARKER: + backup["retired"] = True + else: + backup["cleanup_only"] = True + continue + if ( + not _runtime_backup_target_name(candidate.name, binary_name) + or not stat.S_ISREG(status.st_mode) + or stat.S_ISLNK(status.st_mode) + or status.st_nlink != 1 + ): + raise RuntimeError( + f"refusing unsafe package-cache backup member: {candidate}" + ) + backup["files"][candidate.name] = (candidate, _file_sha256(candidate)) + return backup + + +def _list_runtime_backups(directory: Path, binary_name: str): + names = sorted( + path.name + for path in directory.iterdir() + if _RUNTIME_BACKUP_RE.fullmatch(path.name) is not None + ) + return [ + _inspect_runtime_backup(directory, name, binary_name) for name in names + ] + + +def _current_runtime_targets(directory: Path, binary_name: str): + targets = {} + for target in sorted(directory.iterdir(), key=lambda path: path.name): + if not _runtime_backup_target_name(target.name, binary_name): + continue + status = target.lstat() + if ( + not stat.S_ISREG(status.st_mode) + or stat.S_ISLNK(status.st_mode) + or status.st_nlink != 1 + ): + raise RuntimeError(f"refusing unsafe package-cache target: {target}") + targets[target.name] = target + return targets + + +def _runtime_set_ready_at( + directory: Path, binary_name: str, variant: str, verifier=None +) -> bool: + if _runtime_set_names(directory, binary_name, variant) is None: return False + if verifier is not None: + try: + verifier(directory / binary_name) + except (OSError, RuntimeError): + return False return True +def _cleanup_runtime_backup(backup, lock): + _assert_runtime_lock_owner(lock) + if not backup["cleanup_only"]: + _create_runtime_backup_marker( + backup["path"], _RUNTIME_BACKUP_CLEANUP_MARKER + ) + backup["cleanup_only"] = True + for name in sorted(backup["files"]): + _assert_runtime_lock_owner(lock) + member, digest = backup["files"][name] + if not _path_matches_digest(member, digest): + raise RuntimeError( + f"package-cache backup member changed during cleanup: {name}" + ) + _assert_runtime_lock_owner(lock) + member.unlink() + if backup["retired"]: + _assert_runtime_lock_owner(lock) + (backup["path"] / _RUNTIME_BACKUP_RETIRED_MARKER).unlink() + _assert_runtime_lock_owner(lock) + (backup["path"] / _RUNTIME_BACKUP_CLEANUP_MARKER).unlink() + backup["path"].rmdir() + + +def _copy_runtime_backup_file( + member: Path, target: Path, expected_digest: str, executable: bool, lock +): + descriptor, staged_name = tempfile.mkstemp( + dir=str(target.parent), prefix=".cbm-runtime-recovery-" + ) + staged = Path(staged_name) + last_refresh = time.monotonic() + try: + with member.open("rb") as source, os.fdopen(descriptor, "wb") as output: + descriptor = None + while True: + block = source.read(1024 * 1024) + if not block: + break + output.write(block) + if time.monotonic() - last_refresh >= _RUNTIME_LOCK_HEARTBEAT_SECONDS: + _refresh_runtime_lock(lock) + last_refresh = time.monotonic() + output.flush() + os.fsync(output.fileno()) + staged.chmod(0o755 if executable else 0o644) + if _file_sha256(staged) != expected_digest: + raise RuntimeError( + "package-cache backup changed while it was being restored" + ) + _assert_runtime_lock_owner(lock) + os.replace(staged, target) + finally: + if descriptor is not None: + os.close(descriptor) + try: + staged.unlink() + except FileNotFoundError: + pass + + +def _reconcile_runtime_backups( + directory: Path, binary_name: str, variant: str, verifier, lock +): + _refresh_runtime_lock(lock) + backups = _list_runtime_backups(directory, binary_name) + for backup in [item for item in backups if item["cleanup_only"]]: + _cleanup_runtime_backup(backup, lock) + backups = [item for item in backups if not item["cleanup_only"]] + if not backups: + return + + if _runtime_set_ready_at(directory, binary_name, variant, verifier): + for backup in backups: + _cleanup_runtime_backup(backup, lock) + return + if not backups: + return + + for backup in [ + item + for item in backups + if not item["retired"] and not item["files"] + ]: + _cleanup_runtime_backup(backup, lock) + backups = [ + item for item in backups if item["retired"] or item["files"] + ] + if not backups: + return + if len(backups) != 1: + raise RuntimeError( + "multiple package-cache backup transactions require manual recovery" + ) + + backup = backups[0] + targets = _current_runtime_targets(directory, binary_name) + if not backup["retired"]: + for name, (_, digest) in backup["files"].items(): + target = targets.get(name) + if target is not None and not _path_matches_digest(target, digest): + raise RuntimeError( + f"package-cache backup conflicts with current target: {name}" + ) + else: + binary_target = targets.pop(binary_name, None) + if binary_target is not None: + _assert_runtime_lock_owner(lock) + binary_target.unlink() + for name in sorted(targets): + _assert_runtime_lock_owner(lock) + targets[name].unlink() + + restore_names = sorted( + name for name in backup["files"] if name != binary_name + ) + if binary_name in backup["files"]: + restore_names.append(binary_name) + for name in restore_names: + member, digest = backup["files"][name] + target = directory / name + if not _path_matches_digest(target, digest): + _copy_runtime_backup_file( + member, target, digest, name == binary_name, lock + ) + for name, (_, digest) in backup["files"].items(): + if not _path_matches_digest(directory / name, digest): + raise RuntimeError( + f"package-cache backup restoration failed: {name}" + ) + _cleanup_runtime_backup(backup, lock) + + +def _path_matches_digest(path: Path, expected: str) -> bool: + try: + return _regular_file(path) and _file_sha256(path) == expected + except OSError: + return False + + +def _publish_runtime_set( + staged_paths, + dest: Path, + binary_name: str, + variant: str, + replace_file=None, + verifier=None, +): + """Publish sidecars first, retaining exact-byte backups for recovery.""" + runtime_names = _staged_runtime_names(staged_paths, binary_name, variant) + if replace_file is None: + replace_file = os.replace + _require_safe_runtime_directory(dest.parent) + initial_fingerprint = _runtime_set_fingerprint( + dest.parent, binary_name, variant + ) + lock, contended = _acquire_runtime_lock(dest.parent) + operation_error = None + try: + _refresh_runtime_lock(lock) + _reconcile_runtime_backups( + dest.parent, binary_name, variant, verifier, lock + ) + # Preserve a complete set that appeared or changed while this publisher + # waited. An unchanged pre-existing set may still be repaired/replaced, + # with its bytes retained below for rollback. + locked_fingerprint = _runtime_set_fingerprint( + dest.parent, binary_name, variant + ) + if ( + locked_fingerprint is not None + and (contended or locked_fingerprint != initial_fingerprint) + ): + return + + backup_directory = None + try: + backup_directory = _create_runtime_backup_directory(dest.parent) + pack_like = sorted( + path.name + for path in dest.parent.iterdir() + if path.name.startswith("cbm-ui-") + and path.name.endswith(".pack") + ) + retire_names = (binary_name, _INTEGRATIONS_NAME, *pack_like) + retired = set() + # Retire the executable first: until the final rename, readiness is false. + for name in retire_names: + _assert_runtime_lock_owner(lock) + if name in retired: + continue + retired.add(name) + target = dest.parent / name + try: + status = target.lstat() + except FileNotFoundError: + continue + if ( + not stat.S_ISREG(status.st_mode) + or stat.S_ISLNK(status.st_mode) + or status.st_nlink != 1 + ): + raise RuntimeError( + f"refusing unsafe package-cache target: {target}" + ) + replace_file(target, backup_directory / name) + _assert_runtime_lock_owner(lock) + _create_runtime_backup_marker( + backup_directory, _RUNTIME_BACKUP_RETIRED_MARKER + ) + + for name in runtime_names: + _assert_runtime_lock_owner(lock) + target = dest.parent / name + try: + replace_file(staged_paths[name], target) + staged_paths.pop(name) + except OSError as publish_error: + if not _files_equal_sha256(staged_paths[name], target): + raise publish_error + if _runtime_set_names(dest.parent, binary_name, variant) is None: + raise RuntimeError("published package cache is incomplete") + if verifier is not None: + verifier(dest.parent / binary_name) + _assert_runtime_lock_owner(lock) + except Exception as publication_error: + _assert_runtime_lock_owner(lock) + # Preserve a complete non-cooperating winner; never delete its bytes. + preserve_winner = _runtime_set_ready_at( + dest.parent, binary_name, variant, verifier + ) + if backup_directory is not None: + try: + _reconcile_runtime_backups( + dest.parent, binary_name, variant, verifier, lock + ) + except Exception as recovery_error: + raise RuntimeError( + f"{publication_error}; package-cache recovery failed: " + f"{recovery_error}" + ) from recovery_error + if preserve_winner: + return + raise + else: + backup = _inspect_runtime_backup( + dest.parent, backup_directory.name, binary_name + ) + _cleanup_runtime_backup(backup, lock) + except Exception as exc: + operation_error = exc + raise + finally: + try: + _release_runtime_lock(lock) + except Exception: + if operation_error is None: + raise + + def _download(version: str) -> Path: os_name = _os_name() arch = _arch() ext = "zip" if os_name == "windows" else "tar.gz" + selected_variant = _variant() # Linux ships a fully-static "-portable" build; the standard linux binary # dynamically links glibc 2.38+ and fails on older distros. macOS/Windows # have no such variant. Keep in sync with install.sh / install.js / cli.c. - variant = "-portable" if os_name == "linux" else "" - # Opt into the UI build (embedded graph visualization) with CBM_VARIANT=ui. - # Default is the standard (headless) build. Mirrors install.sh --ui. - ui = "ui-" if os.environ.get("CBM_VARIANT", "").lower() == "ui" else "" - archive = f"codebase-memory-mcp-{ui}{os_name}-{arch}{variant}.{ext}" + portable = "-portable" if os_name == "linux" else "" + # Opt into the UI build, whose verified asset pack is published beside the + # binary, with CBM_VARIANT=ui. Default is the standard (headless) build. + ui = "ui-" if selected_variant == "ui" else "" + archive = f"codebase-memory-mcp-{ui}{os_name}-{arch}{portable}.{ext}" url = f"https://github.com/{REPO}/releases/download/v{version}/{archive}" _validate_url_scheme(url) - dest = _bin_path(version) + dest = _bin_path(version, selected_variant) dest.parent.mkdir(parents=True, exist_ok=True) print( @@ -394,45 +1206,65 @@ def _download(version: str) -> Path: bin_name = ( _WINDOWS_BINARY_NAME if os_name == "windows" else "codebase-memory-mcp" ) - extraction_names = (bin_name,) - cache_names = extraction_names - publish_names = cache_names + extraction_names = (bin_name, _INTEGRATIONS_NAME) + archive_names = ( + _WINDOWS_ARCHIVE_NAMES + if os_name == "windows" + else _UNIX_ARCHIVE_NAMES + ) if ext == "tar.gz": import tarfile with tarfile.open(tmp_archive) as tf: - _safe_extract_tar(tf, tmp) + runtime_names = _safe_extract_tar( + tf, + tmp, + archive_names, + extraction_names, + selected_variant == "ui", + ) else: import zipfile with zipfile.ZipFile(tmp_archive) as zf: - _safe_extract_zip( + runtime_names = _safe_extract_zip( zf, tmp, - _WINDOWS_ARCHIVE_NAMES, + archive_names, extraction_names, + selected_variant == "ui", ) extracted_paths = {} - for name in extraction_names: + for name in runtime_names: extracted_path = Path(tmp) / name - if not extracted_path.is_file(): + if not _regular_file(extracted_path): sys.exit( - f"codebase-memory-mcp: required binary not found after " + f"codebase-memory-mcp: required runtime member not found after " f"extraction: {name}" ) - current = extracted_path.stat().st_mode - extracted_path.chmod( - current | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - ) + if name == bin_name: + current = extracted_path.stat().st_mode + extracted_path.chmod( + current | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) extracted_paths[name] = extracted_path + if _runtime_set_names(Path(tmp), bin_name, selected_variant) is None: + sys.exit( + "codebase-memory-mcp: extracted runtime set failed content " + "verification" + ) try: _verify_candidate(extracted_paths[bin_name]) - except RuntimeError as exc: + except (OSError, RuntimeError) as exc: sys.exit(f"codebase-memory-mcp: {exc}") staged_paths = {} try: - for name in cache_names: - staged_suffix = ".tmp.exe" if sys.platform == "win32" else ".tmp" + for name in runtime_names: + staged_suffix = ( + ".tmp.exe" + if sys.platform == "win32" and name == bin_name + else ".tmp" + ) with tempfile.NamedTemporaryFile( dir=str(dest.parent), prefix=f".{name}.", @@ -441,31 +1273,24 @@ def _download(version: str) -> Path: ) as staged: staged_path = Path(staged.name) shutil.copy2(extracted_paths[name], staged_path) - staged_mode = staged_path.stat().st_mode - staged_path.chmod( - staged_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - ) + if name == bin_name: + staged_mode = staged_path.stat().st_mode + staged_path.chmod( + staged_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) staged_paths[name] = staged_path - if os_name != "windows": - _verify_candidate(staged_paths[bin_name]) - for name in publish_names: - target = dest.parent / name - try: - os.replace(staged_paths[name], target) - staged_paths.pop(name) - except OSError as publish_error: - # Never unlink a destination here: it may belong to a - # concurrent contender. A collision remains eligible only - # when its bytes match this authenticated stage; pair - # execution is checked after payload is published last. - if not _files_equal_sha256(staged_paths[name], target): - raise publish_error - if os_name == "windows": - _verify_candidate(dest.parent / _WINDOWS_BINARY_NAME) - else: - _verify_candidate(dest) - except RuntimeError as exc: + # A crash during publication leaves an incomplete cache, which + # readiness rejects and the next invocation repairs. + _publish_runtime_set( + staged_paths, + dest, + bin_name, + selected_variant, + verifier=_verify_candidate, + ) + _verify_candidate(dest) + except (OSError, RuntimeError) as exc: sys.exit(f"codebase-memory-mcp: {exc}") finally: for staged_path in staged_paths.values(): @@ -480,12 +1305,18 @@ def _download(version: str) -> Path: def main() -> None: version = _version() bin_path = _bin_path(version) + mutation = _runtime_mutation_action(sys.argv[1:]) + if mutation == "update": + print( + "This PyPI copy is maintained by pip. Update it with " + '"python -m pip install --upgrade codebase-memory-mcp". ' + "For a managed standalone installation, run " + '"codebase-memory-mcp install --yes".', + file=sys.stderr, + ) + raise SystemExit(2) - cache_ready = ( - _windows_binary_ready(version) - if sys.platform == "win32" - else bin_path.is_file() - ) + cache_ready = _runtime_set_ready_locked(version) if not cache_ready: bin_path = _download(version) @@ -495,41 +1326,101 @@ def main() -> None: # element as a discrete argv entry — no shell interpretation, no # injection vector. sys.argv forwarding is the whole point of this # shim, so tainted-input suppression is intentional. - args = [str(execution_path)] + sys.argv[1:] + forwarded = _native_args(sys.argv[1:]) + args = [str(execution_path)] + forwarded + + if mutation in ("install", "uninstall"): + result = _run_cache_sensitive_mutation(version, execution_path, forwarded) + if sys.platform == "win32" and result != 0 and mutation == "uninstall": + print( + 'This PyPI Windows copy is portable. Use ' + '"python -m pip uninstall codebase-memory-mcp" for package maintenance. ' + "To create or repair a managed standalone installation, run " + '"codebase-memory-mcp install --yes".', + file=sys.stderr, + ) + raise SystemExit(result) if sys.platform != "win32": os.execv(str(execution_path), args) # noqa: S606 — list form, no shell else: result = subprocess.run(args) # noqa: S603 — list form, no shell=True - mutation = _portable_mutation_action(sys.argv[1:]) - if result.returncode != 0 and mutation is not None: - package_command = ( - "python -m pip install --upgrade codebase-memory-mcp" - if mutation == "update" - else "python -m pip uninstall codebase-memory-mcp" - ) - print( - f'This PyPI Windows copy is portable. Use "{package_command}" ' - f'for package maintenance, or run "codebase-memory-mcp ' - f'install --yes" once to create a managed launcher with ' - f'coordinated self-update/uninstall.', - file=sys.stderr, - ) sys.exit(result.returncode) -def _portable_mutation_action(args): +def _runtime_mutation_action(args): for argument in args: - if argument in ( - "cli", - "hook-augment", - "config", - "install", - "--help", - "-h", - "--version", - ): + if argument in ("--help", "-h", "--version"): return None - if argument in ("update", "uninstall"): + if argument.startswith("-"): + continue + if argument in ("install", "update", "uninstall"): return argument + if argument in ("cli", "hook-augment", "config"): + return None + return None return None + + +def _default_managed_install_dir() -> Path: + if sys.platform != "win32": + return Path.home() / ".local" / "bin" + local_app_data = Path( + os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local") + ) + return local_app_data / "Programs" / "codebase-memory-mcp" + + +def _native_args(args): + result = list(args) + if _runtime_mutation_action(result) == "uninstall" and not any( + argument == "--dir" or argument.startswith("--dir=") + for argument in result + ): + result.extend(("--dir", str(_default_managed_install_dir()))) + return result + + +def _run_cache_sensitive_mutation(version: str, executable: Path, args) -> int: + directory = executable.parent + _require_safe_runtime_directory(directory) + lock, _ = _acquire_runtime_lock(directory) + stop = threading.Event() + heartbeat_errors = [] + + def heartbeat(): + while not stop.wait(_RUNTIME_LOCK_HEARTBEAT_SECONDS): + try: + _refresh_runtime_lock(lock) + except Exception as exc: # surfaced on the launcher thread + heartbeat_errors.append(exc) + return + + try: + _refresh_runtime_lock(lock) + if not _runtime_set_ready(version, _verify_candidate): + raise RuntimeError( + "cached runtime assets changed before mutation launch" + ) + thread = threading.Thread( + name="cbm-cache-lock-heartbeat", target=heartbeat, daemon=True + ) + thread.start() + process = subprocess.Popen( # noqa: S603 — argv list, no shell + [str(executable), *args] + ) + while process.poll() is None: + if heartbeat_errors: + process.terminate() + process.wait() + raise RuntimeError( + f"package-cache lock lost during mutation: " + f"{heartbeat_errors[0]}" + ) + time.sleep(0.05) + return process.returncode + finally: + stop.set() + if "thread" in locals(): + thread.join(timeout=1) + _release_runtime_lock(lock) diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index 4fe2976898..8590280b29 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -1,6 +1,17 @@ +import hashlib +import json +import os +import stat +import subprocess import sys +import tarfile +import tempfile +import threading +import time import unittest +import zipfile from pathlib import Path +from unittest import mock PACKAGE_ROOT = Path(__file__).resolve().parents[1] @@ -22,14 +33,811 @@ def test_non_windows_executes_the_cached_binary(self): self.assertEqual(_cli._execution_path(binary, "linux"), binary) - def test_portable_mutation_guidance_classification_is_preserved(self): - self.assertEqual(_cli._portable_mutation_action(["update"]), "update") + def test_cache_sensitive_mutation_classification_is_explicit(self): + self.assertEqual(_cli._runtime_mutation_action(["update"]), "update") self.assertEqual( - _cli._portable_mutation_action(["uninstall", "--yes"]), + _cli._runtime_mutation_action(["uninstall", "--yes"]), "uninstall", ) - self.assertIsNone(_cli._portable_mutation_action(["install", "--yes"])) - self.assertIsNone(_cli._portable_mutation_action(["cli", "update"])) + self.assertEqual( + _cli._runtime_mutation_action(["install", "--yes"]), "install" + ) + self.assertIsNone(_cli._runtime_mutation_action(["cli", "update"])) + self.assertIsNone(_cli._runtime_mutation_action(["daemon", "update"])) + + def test_wrapper_uninstall_never_defaults_to_its_cache_binary(self): + with mock.patch.object(Path, "home", return_value=Path("/home/user")): + self.assertEqual( + _cli._native_args(["uninstall", "--yes"]), + ["uninstall", "--yes", "--dir", "/home/user/.local/bin"], + ) + self.assertEqual( + _cli._native_args(["uninstall", "--dir", "/custom/bin"]), + ["uninstall", "--dir", "/custom/bin"], + ) + + def test_standard_and_ui_cache_paths_do_not_collide(self): + with mock.patch.dict("os.environ", {"CBM_VARIANT": "standard"}): + standard = _cli._bin_path("0.8.1") + with mock.patch.dict("os.environ", {"CBM_VARIANT": "ui"}): + ui = _cli._bin_path("0.8.1") + + self.assertNotEqual(standard, ui) + + def test_release_archives_require_the_integrations_sidecar(self): + self.assertIn("cbm-integrations.json", _cli._WINDOWS_ARCHIVE_NAMES) + + def test_update_guidance_uses_pip_and_names_managed_install(self): + with mock.patch.object(_cli.sys, "argv", ["cbm", "update"]), \ + mock.patch.object(_cli, "_version", return_value="0.8.1"), \ + mock.patch("sys.stderr") as stderr: + with self.assertRaisesRegex(SystemExit, "2"): + _cli.main() + + output = "".join(call.args[0] for call in stderr.write.call_args_list) + self.assertIn( + "python -m pip install --upgrade codebase-memory-mcp", output + ) + self.assertIn("codebase-memory-mcp install --yes", output) + self.assertNotIn("install.sh", output) + + +class RuntimeSetTests(unittest.TestCase): + def test_candidate_probe_authenticates_adjacent_runtime_assets(self): + with mock.patch.object(_cli.subprocess, "run") as run: + run.return_value.returncode = 0 + _cli._verify_candidate(Path("/tmp/codebase-memory-mcp")) + + self.assertEqual( + run.call_args.args[0], + ["/tmp/codebase-memory-mcp", "--verify-runtime-assets"], + ) + + def test_runtime_readiness_rejects_multiply_linked_cache_leaves(self): + with tempfile.TemporaryDirectory() as root, mock.patch.object( + _cli, "_cache_dir", return_value=Path(root) + ): + binary = _cli._bin_path("0.8.1", "standard") + binary.parent.mkdir(parents=True) + binary.write_text("binary") + integrations = binary.parent / _cli._INTEGRATIONS_NAME + integrations.write_text("integrations") + os.link(integrations, binary.parent / "integrations-hardlink") + + self.assertFalse(_cli._runtime_set_ready("0.8.1")) + + def test_orphan_reconciliation_rejects_multiply_linked_backup_members(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary = directory / "codebase-memory-mcp" + backup = directory / f".cbm-runtime-backup-{'a' * 32}" + backup.mkdir() + (backup / ".retirement-complete").write_bytes(b"") + member = backup / binary.name + member.write_text("binary:old") + os.link(member, directory / "backup-hardlink-copy") + + with mock.patch.object( + _cli, "_bin_path", return_value=binary + ), mock.patch.object( + _cli, "_variant", return_value="standard" + ), self.assertRaisesRegex( + RuntimeError, "unsafe package-cache backup member" + ): + _cli._runtime_set_ready_locked("test-version") + + self.assertTrue(backup.exists(), "unsafe backup was mutated") + self.assertFalse((directory / _cli._RUNTIME_LOCK_NAME).exists()) + + def test_stalled_lock_creator_never_deletes_successor(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + successor = None + + def publish_successor_then_abort(): + nonlocal successor + _cli._runtime_lock_claim_observer = None + successor, _ = _cli._acquire_runtime_lock(directory) + raise RuntimeError("injected stalled creator abort") + + with mock.patch.object( + _cli, + "_runtime_lock_claim_observer", + side_effect=publish_successor_then_abort, + ), self.assertRaisesRegex(RuntimeError, "stalled creator abort"): + _cli._acquire_runtime_lock(directory) + + self.assertIsNotNone(successor) + self.assertTrue(successor[0].exists()) + _cli._release_runtime_lock(successor) + + def test_expired_lease_never_reclaims_live_owner(self): + with tempfile.TemporaryDirectory() as root: + lock_path = Path(root) / _cli._RUNTIME_LOCK_NAME + owner = { + "pid": os.getpid(), + "token": "a" * 32, + "lease_expires_ms": int((time.time() - 1) * 1000), + } + owner_record = f"{json.dumps(owner)}\n" + lock_path.write_text(owner_record) + before = lock_path.stat() + + with mock.patch.object(_cli, "_process_is_alive", return_value=True): + self.assertEqual( + _cli._try_reclaim_runtime_lock(lock_path, "b" * 32), + (False, False), + ) + after = lock_path.stat() + self.assertEqual(lock_path.read_text(), owner_record) + self.assertEqual(json.loads(lock_path.read_text())["token"], owner["token"]) + self.assertEqual(after.st_dev, before.st_dev) + self.assertEqual(after.st_ino, before.st_ino) + + def test_stale_legacy_lock_never_reclaims_live_owner(self): + with tempfile.TemporaryDirectory() as root: + lock_path = Path(root) / _cli._RUNTIME_LOCK_NAME + owner_record = json.dumps( + {"pid": os.getpid(), "token": "a" * 32} + ) + "\n" + lock_path.mkdir() + (lock_path / "owner.json").write_text(owner_record) + stale = time.time() - _cli._RUNTIME_LEGACY_STALE_SECONDS - 1 + os.utime(lock_path, (stale, stale)) + before = lock_path.stat() + + with mock.patch.object(_cli, "_process_is_alive", return_value=True): + self.assertEqual( + _cli._try_reclaim_runtime_lock(lock_path, "b" * 32), + (False, False), + ) + after = lock_path.stat() + self.assertEqual((lock_path / "owner.json").read_text(), owner_record) + self.assertEqual(after.st_dev, before.st_dev) + self.assertEqual(after.st_ino, before.st_ino) + + def test_stale_ownerless_lock_is_reclaimed(self): + with tempfile.TemporaryDirectory() as root: + lock_path = Path(root) / _cli._RUNTIME_LOCK_NAME + lock_path.write_text("") + stale = time.time() - _cli._RUNTIME_OWNERLESS_STALE_SECONDS - 1 + os.utime(lock_path, (stale, stale)) + + self.assertEqual( + _cli._try_reclaim_runtime_lock(lock_path, "b" * 32), + (True, False), + ) + self.assertFalse(lock_path.exists()) + + def test_archive_namespace_is_exact_and_ui_has_one_pack(self): + pack = f"cbm-ui-{'a' * 64}.pack" + names = (*_cli._WINDOWS_ARCHIVE_NAMES, pack) + + self.assertEqual( + _cli._validate_archive_names(names, _cli._WINDOWS_ARCHIVE_NAMES, True), + pack, + ) + with self.assertRaises(SystemExit): + _cli._validate_archive_names( + (*names, "unexpected-root-file"), + _cli._WINDOWS_ARCHIVE_NAMES, + True, + ) + with self.assertRaises(SystemExit): + _cli._validate_archive_names( + (*names, f"cbm-ui-{'b' * 64}.pack"), + _cli._WINDOWS_ARCHIVE_NAMES, + True, + ) + with self.assertRaises(SystemExit): + _cli._validate_archive_names( + names, _cli._WINDOWS_ARCHIVE_NAMES, False + ) + + def test_ui_zip_extracts_only_the_runtime_set(self): + pack = f"cbm-ui-{'a' * 64}.pack" + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.zip" + destination = Path(root) / "extract" + destination.mkdir() + with zipfile.ZipFile(archive, "w") as zf: + for name in (*_cli._WINDOWS_ARCHIVE_NAMES, pack): + zf.writestr(name, name.encode()) + + with zipfile.ZipFile(archive) as zf: + names = _cli._safe_extract_zip( + zf, + str(destination), + _cli._WINDOWS_ARCHIVE_NAMES, + (_cli._WINDOWS_BINARY_NAME, _cli._INTEGRATIONS_NAME), + True, + ) + + self.assertEqual( + names, + (_cli._WINDOWS_BINARY_NAME, _cli._INTEGRATIONS_NAME, pack), + ) + self.assertEqual( + {path.name for path in destination.iterdir()}, set(names) + ) + + def test_unix_tar_rejects_an_unexpected_root_member(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.tar.gz" + with tarfile.open(archive, "w:gz") as tf: + for name in (*_cli._UNIX_ARCHIVE_NAMES, "unexpected-root-file"): + data_path = Path(root) / f"fixture-{len(tf.getmembers())}" + data_path.write_bytes(name.encode()) + tf.add(data_path, arcname=name) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf, self.assertRaises(SystemExit): + _cli._safe_extract_tar( + tf, + str(destination), + _cli._UNIX_ARCHIVE_NAMES, + ("codebase-memory-mcp", _cli._INTEGRATIONS_NAME), + False, + ) + + def test_unix_tar_rejects_hardlink_members(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.tar.gz" + with tarfile.open(archive, "w:gz") as tf: + info = tarfile.TarInfo("codebase-memory-mcp") + info.type = tarfile.LNKTYPE + info.linkname = "cbm-integrations.json" + tf.addfile(info) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf, self.assertRaises(SystemExit): + _cli._safe_extract_tar( + tf, + str(destination), + ("codebase-memory-mcp",), + ("codebase-memory-mcp",), + False, + ) + + def test_windows_zip_rejects_symlink_metadata(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.zip" + with zipfile.ZipFile(archive, "w") as zf: + info = zipfile.ZipInfo(_cli._WINDOWS_BINARY_NAME) + info.create_system = 3 + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + zf.writestr(info, "cbm-integrations.json") + destination = Path(root) / "extract" + destination.mkdir() + with zipfile.ZipFile(archive) as zf, self.assertRaises(SystemExit): + _cli._safe_extract_zip( + zf, + str(destination), + (_cli._WINDOWS_BINARY_NAME,), + (_cli._WINDOWS_BINARY_NAME,), + False, + ) + + def test_readiness_requires_the_complete_selected_variant(self): + with tempfile.TemporaryDirectory() as root, mock.patch.object( + _cli, "_cache_dir", return_value=Path(root) + ): + with mock.patch.dict(os.environ, {"CBM_VARIANT": "standard"}): + standard_binary = _cli._bin_path("0.8.1") + standard_binary.parent.mkdir(parents=True) + standard_binary.write_text("binary") + (standard_binary.parent / _cli._INTEGRATIONS_NAME).write_text( + "integrations" + ) + self.assertTrue(_cli._runtime_set_ready("0.8.1")) + (standard_binary.parent / _cli._INTEGRATIONS_NAME).unlink() + self.assertFalse(_cli._runtime_set_ready("0.8.1")) + + with mock.patch.dict(os.environ, {"CBM_VARIANT": "ui"}): + ui_binary = _cli._bin_path("0.8.1") + ui_binary.parent.mkdir(parents=True) + ui_binary.write_text("binary") + (ui_binary.parent / _cli._INTEGRATIONS_NAME).write_text( + "integrations" + ) + pack_bytes = b"pack" + pack_name = ( + f"cbm-ui-{hashlib.sha256(pack_bytes).hexdigest()}.pack" + ) + (ui_binary.parent / pack_name).write_bytes(pack_bytes) + self.assertTrue(_cli._runtime_set_ready("0.8.1")) + second_pack = f"cbm-ui-{'b' * 64}.pack" + (ui_binary.parent / second_pack).write_text("second pack") + self.assertFalse(_cli._runtime_set_ready("0.8.1")) + + def test_ui_readiness_rejects_digest_mismatched_pack_bytes(self): + with tempfile.TemporaryDirectory() as root, mock.patch.object( + _cli, "_cache_dir", return_value=Path(root) + ), mock.patch.dict(os.environ, {"CBM_VARIANT": "ui"}): + binary = _cli._bin_path("0.8.1") + binary.parent.mkdir(parents=True) + binary.write_text("binary") + (binary.parent / _cli._INTEGRATIONS_NAME).write_text("integrations") + authenticated = b"authenticated pack" + digest = hashlib.sha256(authenticated).hexdigest() + pack = binary.parent / f"cbm-ui-{digest}.pack" + pack.write_bytes(b"corrupt pack") + + self.assertFalse( + _cli._runtime_set_ready("0.8.1"), + "a well-shaped filename must not authenticate different bytes", + ) + + def test_ui_publication_rejects_digest_mismatched_staged_pack(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary_name = "codebase-memory-mcp" + authenticated = b"authenticated pack" + pack = ( + f"cbm-ui-{hashlib.sha256(authenticated).hexdigest()}.pack" + ) + staged_paths = {} + for name, contents in ( + (binary_name, b"binary"), + (_cli._INTEGRATIONS_NAME, b"integrations"), + (pack, b"corrupt pack"), + ): + staged = directory / f".stage-{name}" + staged.write_bytes(contents) + staged_paths[name] = staged + + with self.assertRaisesRegex(RuntimeError, "digest"): + _cli._publish_runtime_set( + staged_paths, directory / binary_name, binary_name, "ui" + ) + + self.assertFalse((directory / binary_name).exists()) + self.assertFalse((directory / _cli._RUNTIME_LOCK_NAME).exists()) + + def test_publication_commits_sidecars_before_binary(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary_name = "codebase-memory-mcp" + pack_bytes = b"pack" + pack = f"cbm-ui-{hashlib.sha256(pack_bytes).hexdigest()}.pack" + staged_paths = {} + for name, contents in ( + (binary_name, b"binary"), + (_cli._INTEGRATIONS_NAME, b"integrations"), + (pack, pack_bytes), + ): + staged = directory / f"stage-{name}" + staged.write_bytes(contents) + staged_paths[name] = staged + committed = [] + replace = os.replace + + def record_replace(source, destination): + committed.append(Path(destination).name) + replace(source, destination) + + with mock.patch.object(_cli.os, "replace", side_effect=record_replace): + _cli._publish_runtime_set( + staged_paths, directory / binary_name, binary_name, "ui" + ) + + self.assertEqual( + committed, [_cli._INTEGRATIONS_NAME, pack, binary_name] + ) + + def test_killed_publisher_is_reconciled_by_locked_readiness(self): + helper_source = r""" +import os +import sys +import time +from pathlib import Path + +source_root, directory_raw, staged_raw, marker_raw, crash_name = sys.argv[1:] +sys.path.insert(0, source_root) +from codebase_memory_mcp import _cli + +directory = Path(directory_raw) +staged = Path(staged_raw) +marker = Path(marker_raw) +binary_name = "codebase-memory-mcp" +staged_paths = { + _cli._INTEGRATIONS_NAME: staged / _cli._INTEGRATIONS_NAME, + binary_name: staged / binary_name, +} +replace = os.replace + +def pausing_replace(source, destination): + replace(source, destination) + if Path(destination) == directory / crash_name: + marker.write_text("reached\n") + while True: + time.sleep(60) + +def verifier(binary): + if not Path(binary).read_text().startswith("binary:"): + raise RuntimeError("test binary failed verification") + +_cli._publish_runtime_set( + staged_paths, + directory / binary_name, + binary_name, + "standard", + replace_file=pausing_replace, + verifier=verifier, +) +""" + binary_name = "codebase-memory-mcp" + for crash_name, expected_ready in ( + (_cli._INTEGRATIONS_NAME, False), + (binary_name, True), + ): + with self.subTest(crash_name=crash_name), tempfile.TemporaryDirectory() as root: + root_path = Path(root) + directory = root_path / "destination" + staged = root_path / "staged" + marker = root_path / "crash-reached" + directory.mkdir() + staged.mkdir() + (directory / binary_name).write_text("corrupt:old") + (directory / _cli._INTEGRATIONS_NAME).write_text( + "integrations:old" + ) + (staged / binary_name).write_text("binary:candidate") + (staged / _cli._INTEGRATIONS_NAME).write_text( + "integrations:candidate" + ) + environment = os.environ.copy() + environment["PYTHONPYCACHEPREFIX"] = str(root_path / "pycache") + process = subprocess.Popen( + [ + sys.executable, + "-c", + helper_source, + str(PACKAGE_ROOT / "src"), + str(directory), + str(staged), + str(marker), + crash_name, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + env=environment, + ) + try: + deadline = time.monotonic() + 10 + while not marker.exists() and time.monotonic() < deadline: + if process.poll() is not None: + stderr = process.communicate()[1] + self.fail( + "publication helper exited before crash gate: " + f"{stderr}" + ) + time.sleep(0.01) + self.assertTrue( + marker.exists(), + "publication helper did not reach crash gate", + ) + backups = [ + path + for path in directory.iterdir() + if path.name.startswith(".cbm-runtime-backup-") + ] + self.assertEqual(len(backups), 1) + self.assertTrue(backups[0].is_dir()) + self.assertTrue( + (backups[0] / ".retirement-complete").is_file() + ) + + process.kill() + process.wait(5) + self.assertNotEqual(process.returncode, 0) + + def verifier(binary): + if not Path(binary).read_text().startswith("binary:"): + raise RuntimeError("test binary failed verification") + + with mock.patch.object( + _cli, "_bin_path", return_value=directory / binary_name + ), mock.patch.object( + _cli, "_variant", return_value="standard" + ), mock.patch.object( + _cli, "_verify_candidate", side_effect=verifier + ): + ready = _cli._runtime_set_ready_locked("test-version") + self.assertEqual(ready, expected_ready) + if not ready: + self.assertEqual( + (directory / binary_name).read_text(), + "corrupt:old", + ) + self.assertEqual( + (directory / _cli._INTEGRATIONS_NAME).read_text(), + "integrations:old", + ) + retry_staged = root_path / "retry-staged" + retry_staged.mkdir() + retry_paths = { + binary_name: retry_staged / binary_name, + _cli._INTEGRATIONS_NAME: retry_staged + / _cli._INTEGRATIONS_NAME, + } + retry_paths[binary_name].write_text("binary:candidate") + retry_paths[_cli._INTEGRATIONS_NAME].write_text( + "integrations:candidate" + ) + _cli._publish_runtime_set( + retry_paths, + directory / binary_name, + binary_name, + "standard", + verifier=verifier, + ) + + self.assertEqual( + (directory / binary_name).read_text(), "binary:candidate" + ) + self.assertEqual( + (directory / _cli._INTEGRATIONS_NAME).read_text(), + "integrations:candidate", + ) + self.assertFalse( + any( + path.name.startswith(".cbm-runtime-backup-") + for path in directory.iterdir() + ) + ) + self.assertFalse( + (directory / _cli._RUNTIME_LOCK_NAME).exists() + ) + finally: + if process.poll() is None: + process.kill() + process.wait(5) + if process.stderr is not None: + process.stderr.close() + + def test_publication_failure_restores_prior_complete_runtime_set(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary_name = "codebase-memory-mcp" + old_pack_bytes = b"old pack" + old_pack = f"cbm-ui-{hashlib.sha256(old_pack_bytes).hexdigest()}.pack" + (directory / binary_name).write_text("binary:old") + (directory / _cli._INTEGRATIONS_NAME).write_text("integrations:old") + (directory / old_pack).write_bytes(old_pack_bytes) + + candidate_pack_bytes = b"candidate pack" + candidate_pack = ( + f"cbm-ui-{hashlib.sha256(candidate_pack_bytes).hexdigest()}.pack" + ) + staged_paths = {} + for name, contents in ( + (binary_name, b"binary:candidate"), + (_cli._INTEGRATIONS_NAME, b"integrations:candidate"), + (candidate_pack, candidate_pack_bytes), + ): + staged = directory / f"stage-{name}" + staged.write_bytes(contents) + staged_paths[name] = staged + + replace = os.replace + failed = False + retired = [] + + def fail_binary_publish(source, destination): + nonlocal failed + source = Path(source) + destination = Path(destination) + if destination.parent.name.startswith(".cbm-runtime-backup-"): + retired.append(source.name) + if destination == directory / binary_name and not failed: + failed = True + raise OSError("injected binary publication failure") + replace(source, destination) + + with self.assertRaises(OSError): + _cli._publish_runtime_set( + staged_paths, + directory / binary_name, + binary_name, + "ui", + replace_file=fail_binary_publish, + ) + + self.assertTrue(failed) + self.assertEqual(retired[0], binary_name) + self.assertEqual((directory / binary_name).read_text(), "binary:old") + self.assertEqual( + (directory / _cli._INTEGRATIONS_NAME).read_text(), + "integrations:old", + ) + self.assertEqual((directory / old_pack).read_bytes(), old_pack_bytes) + self.assertFalse((directory / candidate_pack).exists()) + self.assertFalse( + any( + path.name.startswith(".cbm-runtime-backup-") + for path in directory.iterdir() + ) + ) + + def test_concurrent_publishers_preserve_the_first_complete_winner(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary_name = "codebase-memory-mcp" + + def stages(tag): + pack_bytes = f"pack:{tag}".encode() + pack = ( + f"cbm-ui-{hashlib.sha256(pack_bytes).hexdigest()}.pack" + ) + result = {} + for name, contents in ( + (binary_name, f"binary:{tag}".encode()), + (_cli._INTEGRATIONS_NAME, f"integrations:{tag}".encode()), + (pack, pack_bytes), + ): + staged = directory / f".stage-{tag}-{name}" + staged.write_bytes(contents) + result[name] = staged + return result, pack + + first_stages, first_pack = stages("first") + second_stages, second_pack = stages("second") + first_holds_lock = threading.Event() + allow_first = threading.Event() + second_observed_lock = threading.Event() + errors = [] + replace = os.replace + read_lock_owner = _cli._read_runtime_lock_owner + + def pause_first(source, destination): + replace(source, destination) + if Path(destination) == directory / binary_name: + first_holds_lock.set() + if not allow_first.wait(5): + raise RuntimeError("test did not release first publisher") + + def observe_waiter(lock_path): + if threading.current_thread().name == "publisher-second": + second_observed_lock.set() + return read_lock_owner(lock_path) + + def publish(label, staged_paths, replace_file=None): + try: + _cli._publish_runtime_set( + staged_paths, + directory / binary_name, + binary_name, + "ui", + replace_file=replace_file, + ) + except Exception as exc: # surfaced on the test thread below + errors.append((label, exc)) + + first = threading.Thread( + name="publisher-first", + target=publish, + args=("first", first_stages, pause_first), + ) + second = threading.Thread( + name="publisher-second", + target=publish, + args=("second", second_stages), + ) + with mock.patch.object( + _cli, + "_read_runtime_lock_owner", + side_effect=observe_waiter, + ): + first.start() + try: + self.assertTrue(first_holds_lock.wait(2)) + second.start() + self.assertTrue(second_observed_lock.wait(2)) + finally: + allow_first.set() + first.join(5) + if second.ident is not None: + second.join(5) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertEqual(errors, []) + self.assertEqual( + (directory / binary_name).read_bytes(), b"binary:first" + ) + self.assertEqual( + (directory / _cli._INTEGRATIONS_NAME).read_bytes(), + b"integrations:first", + ) + self.assertEqual( + (directory / first_pack).read_bytes(), b"pack:first" + ) + self.assertFalse((directory / second_pack).exists()) + self.assertFalse((directory / _cli._RUNTIME_LOCK_NAME).exists()) + + def test_failure_never_deletes_a_complete_foreign_winner(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + binary_name = "codebase-memory-mcp" + old_pack_bytes = b"pack:old" + old_pack = ( + f"cbm-ui-{hashlib.sha256(old_pack_bytes).hexdigest()}.pack" + ) + (directory / binary_name).write_bytes(b"binary:old") + (directory / _cli._INTEGRATIONS_NAME).write_bytes( + b"integrations:old" + ) + (directory / old_pack).write_bytes(old_pack_bytes) + + candidate_pack_bytes = b"pack:candidate" + candidate_pack = ( + f"cbm-ui-{hashlib.sha256(candidate_pack_bytes).hexdigest()}.pack" + ) + staged_paths = {} + for name, contents in ( + (binary_name, b"binary:candidate"), + (_cli._INTEGRATIONS_NAME, b"integrations:candidate"), + (candidate_pack, candidate_pack_bytes), + ): + staged = directory / f".stage-candidate-{name}" + staged.write_bytes(contents) + staged_paths[name] = staged + + foreign_pack_bytes = b"pack:foreign" + foreign_pack = ( + f"cbm-ui-{hashlib.sha256(foreign_pack_bytes).hexdigest()}.pack" + ) + replace = os.replace + injected = False + + def install_foreign_winner(source, destination): + nonlocal injected + source = Path(source) + destination = Path(destination) + if destination == directory / binary_name and not injected: + injected = True + for path in directory.iterdir(): + if path.name.startswith("cbm-ui-") and path.name.endswith( + ".pack" + ): + path.unlink() + (directory / _cli._INTEGRATIONS_NAME).write_bytes( + b"integrations:foreign" + ) + (directory / foreign_pack).write_bytes(foreign_pack_bytes) + (directory / binary_name).write_bytes(b"binary:foreign") + raise OSError("simulated non-cooperating winner") + replace(source, destination) + + _cli._publish_runtime_set( + staged_paths, + directory / binary_name, + binary_name, + "ui", + replace_file=install_foreign_winner, + ) + + self.assertTrue(injected) + self.assertEqual( + (directory / binary_name).read_bytes(), b"binary:foreign" + ) + self.assertEqual( + (directory / _cli._INTEGRATIONS_NAME).read_bytes(), + b"integrations:foreign", + ) + self.assertEqual( + (directory / foreign_pack).read_bytes(), foreign_pack_bytes + ) + self.assertFalse((directory / candidate_pack).exists()) + self.assertFalse((directory / old_pack).exists()) + self.assertFalse( + any( + path.name.startswith(".cbm-runtime-backup-") + for path in directory.iterdir() + ) + ) if __name__ == "__main__": diff --git a/pkg/scoop/codebase-memory-mcp.json b/pkg/scoop/codebase-memory-mcp.json index d4853c492f..f4ae23c1b1 100644 --- a/pkg/scoop/codebase-memory-mcp.json +++ b/pkg/scoop/codebase-memory-mcp.json @@ -1,6 +1,6 @@ { "version": "0.8.1", - "description": "Fast code intelligence engine for AI coding agents — single static binary MCP server", + "description": "Fast code intelligence engine — native executable with verified runtime assets", "homepage": "https://github.com/DeusData/codebase-memory-mcp", "license": "MIT", "architecture": { @@ -10,9 +10,6 @@ "bin": "codebase-memory-mcp.exe" } }, - "post_install": [ - "& \"$dir\\codebase-memory-mcp.exe\" install -y 2>&1 | Out-Null" - ], "checkver": { "github": "https://github.com/DeusData/codebase-memory-mcp" }, diff --git a/pkg/winget/manifests/d/DeusData/CodebaseMemoryMcp/0.8.1/DeusData.CodebaseMemoryMcp.locale.en-US.yaml b/pkg/winget/manifests/d/DeusData/CodebaseMemoryMcp/0.8.1/DeusData.CodebaseMemoryMcp.locale.en-US.yaml index 7377238a45..a9440ad0f6 100644 --- a/pkg/winget/manifests/d/DeusData/CodebaseMemoryMcp/0.8.1/DeusData.CodebaseMemoryMcp.locale.en-US.yaml +++ b/pkg/winget/manifests/d/DeusData/CodebaseMemoryMcp/0.8.1/DeusData.CodebaseMemoryMcp.locale.en-US.yaml @@ -10,11 +10,13 @@ License: MIT LicenseUrl: https://github.com/DeusData/codebase-memory-mcp/blob/main/LICENSE ShortDescription: Fast code intelligence engine for AI coding agents Description: |- - codebase-memory-mcp is a single static binary MCP server that indexes codebases - at extreme speed and exposes 14 MCP tools for AI coding agents. Full indexes an - average repository in milliseconds, the Linux kernel (28M LOC) in 3 minutes. - Supports 155 languages via vendored tree-sitter grammars. Zero dependencies. - Plug and play across 11 coding agents including Claude Code, Codex CLI, and Zed. + codebase-memory-mcp is a native MCP server distributed as a verified runtime + set. The portable archive keeps its authenticated release-owned assets with the + executable registered by WinGet. It indexes codebases at extreme speed and + exposes 14 MCP tools for AI coding agents. Full indexes an average repository + in milliseconds, the Linux kernel (28M LOC) in 3 minutes. Supports 155 languages + via vendored tree-sitter grammars. No hosted service or API key. Plug and play + across 11 coding agents including Claude Code, Codex CLI, and Zed. Moniker: codebase-memory-mcp Tags: - mcp diff --git a/scripts/README.md b/scripts/README.md index 9e9a5d42d3..619959ebe2 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,7 +4,7 @@ only a canonical leg script may exercise the product.* Local CI, PR CI, the dry run and the release all call the SAME files below — venues differ only in host specs, architecture and inputs, never in what a leg does. Platform -differences (arm64 sanitizer flags, the Windows launcher, the linux portable +differences (arm64 sanitizer flags, native Windows behavior, the Linux portable binary) live INSIDE these scripts, once, never per-venue. `tests/test_venue_parity_contract.sh` (run as Step 0j of every test leg) turns any violation — inline harness logic in a workflow, a bypassed wrapper, a @@ -17,12 +17,13 @@ codes) and rejects unknown flags with exit 2 + `Please consult --help.` | leg | entry | what a run gives you | |---|---|---| -| **test** | `test.sh` | DEFAULT = the venue leg: static contracts (Step 0a–0j) + CLEAN ASan+UBSan build + all suites via the parallel harness + prod-binary guards. `--suites a,b` = iteration mode (incremental rebuild, subset, seconds). `--tsan` = the ThreadSanitizer leg. CLANGARM64 gets CI's trap-UBSan flags automatically. | -| **build** | `build.sh` | CLEAN production build (+ launcher on Windows). ccache via `env.sh` makes repeats fast; `CCACHE_COMPILERCHECK=content` guarantees a hit is byte-identical to a cold compile — never stale. `--with-ui`, `--version`, `STATIC=1`, `BUILD_DIR=`. | +| **test** | `test.sh` | DEFAULT = the venue leg: static contracts (Step 0a–0r) + CLEAN ASan+UBSan build + all suites via the parallel harness + prod-binary guards. `--suites a,b` = iteration mode (incremental rebuild, subset, seconds). `--tsan` = the ThreadSanitizer leg. CLANGARM64 gets CI's trap-UBSan flags automatically. | +| **package wrappers** | `ci/test-package-wrappers.sh` | Go, npm, and PyPI runtime-set publication/lock suites on the current host. CI runs the same entry on Linux and Windows so both platform lock implementations gate. | +| **build** | `build.sh` | CLEAN production runtime set (native executable + authenticated integration asset; `--with-ui` adds one content-addressed UI pack). ccache via `env.sh` makes repeats fast; `CCACHE_COMPILERCHECK=content` guarantees a hit is byte-identical to a cold compile — never stale. `--version`, `STATIC=1`, `BUILD_DIR=`. | | **lint** | `lint.sh` | clang-tidy + cppcheck + clang-format (+ no-skips policy). `--ci` = the CI gate set (no clang-tidy). Drives the same make targets as `make lint`/`lint-ci`. | -| **smoke (unix)** | `smoke-local.sh` | Stages a full release fixture, serves it on a kernel-assigned port, runs `smoke-test.sh` (ALL phases incl. download/install/update E2E) inside a disposable HOME/XDG/TMP sandbox. `ui` variant makes a missing embedded UI a FAILURE. `CBM_SMOKE_ARTIFACT_DIR` = smoke an extracted release artifact verbatim (release mode). | -| **smoke (windows)** | `../test-infrastructure/vm/vm-smoke.sh` | Same contract for the launcher+payload pair, plus the user-PATH registry guard (prepare/verify/cleanup). | -| **smoke-invariants** | `smoke-invariants.sh` | The "shipped binary does not fail" battery (MCP handshake, all tools invocable, malformed-input resilience) — no fixture server, no install E2E. `smoke.yml` runs it on the WIDEST build-from-source matrix (incl. older-glibc legs the release artifacts cannot cover). | +| **smoke (unix)** | `smoke-local.sh` | Stages a full release fixture, serves it on a kernel-assigned port, runs `smoke-test.sh` (ALL phases incl. download/install/update E2E) inside a disposable HOME/XDG/TMP sandbox. `ui` variant makes a missing verified UI pack a FAILURE. `CBM_SMOKE_ARTIFACT_DIR` = smoke an extracted release artifact verbatim (release mode). | +| **smoke (windows)** | `../test-infrastructure/vm/vm-smoke.sh` | Same verified runtime-set contract on the real Windows VM, plus the user-PATH registry guard (prepare/verify/cleanup). | +| **smoke-invariants** | `smoke-invariants.sh` | Production-path resilience battery (MCP handshake, all tools invocable, malformed-input handling, supervised crash/hang recovery) — no fixture server or install E2E. `smoke.yml` runs an explicitly seam-enabled build on the WIDEST source matrix; release artifacts remain seam-free and use the release-shaped smoke legs above. | | **soak** | `soak-legs.sh` | The release-gating soak SEQUENCE: `quick` then `query-leak` (the #581 detector — never reindexes, so RSS growth = query-path leak), each guarded by a completion-summary check. `--legs quick` for the ASan single-leg variant. Duration is per leg. | Internal harnesses — never called directly by a venue (the contract forbids diff --git a/scripts/build.sh b/scripts/build.sh index 0b44dcbf49..7b6f9daceb 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -3,7 +3,7 @@ # # Usage: # scripts/build.sh # Standard binary -# scripts/build.sh --with-ui # Binary with embedded UI +# scripts/build.sh --with-ui # Binary + external UI asset pack # scripts/build.sh --help # Full usage # scripts/build.sh --version v0.8.0 # With version stamp # scripts/build.sh --arch x86_64 # Force x86_64 build @@ -29,7 +29,7 @@ from current sources; a cache hit is byte-identical to a cold compile by construction (CCACHE_COMPILERCHECK=content). Options: - --with-ui Embed the web UI (builds the frontend first; needs node). + --with-ui Build the web UI as a content-addressed sidecar (needs node). --version V Stamp the version string (release venue passes the tag). --arch ARCH Force target arch (arm64 | x86_64), e.g. under Rosetta. -h, --help This text. @@ -158,7 +158,7 @@ fi # Stage the integration-template asset next to the binary so the dev/CI build # mirrors the release archive layout. The binary resolves cbm-integrations.json -# next to itself, in $CBM_ASSETS_DIR, or under ~/.cbm/assets//; a plain +# next to itself, in $CBM_ASSETS_DIR, or under ~/.cbm/assets//; a plain # build/c binary has none of those, so install/uninstall (which render the # templates to verify hash and ownership) would fail closed exactly as they do # for a user who deleted the file. Copying it here keeps `install` working @@ -166,4 +166,20 @@ fi # then see the same adjacency. cp "$ROOT/assets/cbm-integrations.json" "$BUILD_DIR/cbm-integrations.json" +if $WITH_UI; then + shopt -s nullglob + UI_PACKS=("$BUILD_DIR"/cbm-ui-*.pack) + shopt -u nullglob + if [ "${#UI_PACKS[@]}" -ne 1 ]; then + echo "build.sh: UI build did not produce exactly one content-addressed asset pack" >&2 + exit 1 + fi + UI_PACK_NAME="$(basename "${UI_PACKS[0]}")" + if ! [[ "$UI_PACK_NAME" =~ ^cbm-ui-[0-9a-f]{64}\.pack$ ]]; then + echo "build.sh: invalid UI asset pack name: $UI_PACK_NAME" >&2 + exit 1 + fi + echo "=== UI assets: ${UI_PACKS[0]} ===" +fi + echo "=== Build complete: ${BUILD_DIR}/codebase-memory-mcp ===" diff --git a/scripts/ci/append-vt-notes.sh b/scripts/ci/append-vt-notes.sh index 56f5b42f40..5880461743 100755 --- a/scripts/ci/append-vt-notes.sh +++ b/scripts/ci/append-vt-notes.sh @@ -1,29 +1,255 @@ #!/usr/bin/env bash -# Append the Security Verification section to the release notes: per-binary -# sha256 + VirusTotal links. This step only runs after check-virustotal.sh -# passed, and that gate is ZERO tolerance — any detection blocks the release — -# so "0 detections" here is a verified statement, never an assumption. -# Expects: GH_TOKEN, VERSION; run from the verify job workspace (binaries/). +# Idempotently publish the exact VirusTotal scope and measured result evidence +# after check-virustotal.sh has passed. set -euo pipefail -TABLE="\n\n## Security Verification\n\n" -TABLE+="All release binaries scanned with 70+ antivirus engines — **0 detections**.\n\n" -TABLE+="| Binary | SHA-256 | VirusTotal |\n" -TABLE+="|--------|---------|------------|\n" - -for bin in binaries/codebase-memory-mcp-*; do - [ -f "$bin" ] || continue - name=$(basename "$bin") - sha256=$(sha256sum "$bin" 2>/dev/null | awk '{print $1}' \ - || shasum -a 256 "$bin" | awk '{print $1}') - label=$(echo "$name" | sed 's/^codebase-memory-mcp-//' | sed 's/\.exe$//') - short="${sha256:0:20}..." - vt_url="https://www.virustotal.com/gui/file/${sha256}/detection" - TABLE+="| \`${label}\` | \`${short}\` | [0 detections ✅](${vt_url}) |\n" -done - -CURRENT=$(gh release view "$VERSION" \ - --json body --jq '.body // ""' --repo "$GITHUB_REPOSITORY") -printf '%s%b' "$CURRENT" "$TABLE" > /tmp/release_notes.md +: "${GH_TOKEN:?append-vt-notes: GH_TOKEN is required}" +: "${VERSION:?append-vt-notes: VERSION is required}" +: "${GITHUB_REPOSITORY:?append-vt-notes: GITHUB_REPOSITORY is required}" + +VT_ASSOCIATIONS="${VT_ASSOCIATIONS:-binaries/associations.tsv}" +VT_RESULTS_PATH="${VT_RESULTS_PATH:-binaries/vt-results.tsv}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/cbm-vt-notes.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT + +gh release view "$VERSION" \ + --json body --jq '.body // ""' --repo "$GITHUB_REPOSITORY" > "$WORK/current.md" + +python3 - "$VT_ASSOCIATIONS" "$VT_RESULTS_PATH" "$WORK/current.md" "$WORK/updated.md" \ + "$GITHUB_REPOSITORY" "$VERSION" <<'PY' +from __future__ import annotations + +import csv +import pathlib +import re +import sys +import urllib.parse +from typing import Dict, List, Sequence, Tuple + + +START = "" +END = "" +ASSOCIATION_FIELDS = ( + "association_type", + "archive", + "archive_sha256", + "variant", + "kind", + "member", + "asset_path", + "mime", + "scan_path", + "object_sha256", + "size", +) +RESULT_FIELDS = ( + "scan_path", + "sha256", + "size", + "association_count", + "completed_engines", + "total_engines", + "malicious", + "suspicious", + "analysis_id", + "microsoft_category", + "microsoft_engine_version", + "microsoft_engine_update", + "virustotal_url", +) + + +def fail(message: str) -> None: + raise SystemExit(f"append-vt-notes: {message}") + + +def read_tsv( + path: pathlib.Path, + marker: str, + fields: Sequence[str], +) -> Tuple[Dict[str, int], List[Dict[str, str]]]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: + fail(f"missing, unsafe or oversized evidence file: {path}") + lines = path.read_text(encoding="utf-8").splitlines() + if not lines or lines[0] != f"# {marker}": + fail(f"wrong evidence marker in {path}") + metadata: Dict[str, int] = {} + cursor = 1 + while cursor < len(lines) and lines[cursor].startswith("# "): + key, separator, raw_value = lines[cursor][2:].partition("=") + if not separator or not key or key in metadata or not raw_value.isdigit(): + fail(f"malformed evidence metadata in {path}") + metadata[key] = int(raw_value) + cursor += 1 + reader = csv.DictReader(lines[cursor:], delimiter="\t") + if tuple(reader.fieldnames or ()) != tuple(fields): + fail(f"unexpected TSV header in {path}") + rows = list(reader) + if any(None in row or any(row.get(field) is None for field in fields) for row in rows): + fail(f"TSV row has missing or surplus cells in {path}") + if not rows: + fail(f"empty evidence rows in {path}") + return metadata, rows + + +association_path = pathlib.Path(sys.argv[1]) +results_path = pathlib.Path(sys.argv[2]) +current_path = pathlib.Path(sys.argv[3]) +updated_path = pathlib.Path(sys.argv[4]) +repository = sys.argv[5] +version = sys.argv[6] +if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) is None or not version: + fail("unsafe repository or version for public evidence links") +asset_base = ( + f"https://github.com/{repository}/releases/download/" + f"{urllib.parse.quote(version, safe='')}" +) +association_meta, associations = read_tsv( + association_path, + "cbm-release-scan-associations-v2", + ASSOCIATION_FIELDS, +) +result_meta, results = read_tsv( + results_path, + "cbm-virustotal-results-v1", + RESULT_FIELDS, +) +if association_meta.get("associations") != len(associations): + fail("association manifest count does not match its rows") +if result_meta.get("scan_objects") != len(results): + fail("VirusTotal result count does not match its rows") +if result_meta.get("associations") != len(associations): + fail("VirusTotal results do not cover the association manifest") + +results_by_path: Dict[str, Dict[str, str]] = {} +association_counts: Dict[str, int] = {} +executable_paths = { + association["scan_path"] + for association in associations + if association["kind"] == "binary" +} +for association in associations: + scan_path = association["scan_path"] + association_counts[scan_path] = association_counts.get(scan_path, 0) + 1 +for result in results: + scan_path = result["scan_path"] + if scan_path in results_by_path: + fail(f"duplicate VirusTotal result path: {scan_path}") + for numeric in ( + "size", + "association_count", + "completed_engines", + "total_engines", + "malicious", + "suspicious", + ): + if not result[numeric].isdigit(): + fail(f"malformed numeric result field {numeric}: {scan_path}") + if int(result["malicious"]) != 0 or int(result["suspicious"]) != 0: + fail(f"detected object reached release-note publication: {scan_path}") + if ( + re.fullmatch(r"[0-9a-f]{64}", result["sha256"]) is None + or result["virustotal_url"] + != f"https://www.virustotal.com/gui/file/{result['sha256']}/detection" + ): + fail(f"result SHA/link is malformed or mismatched: {scan_path}") + if int(result["association_count"]) != association_counts.get(scan_path): + fail(f"result association count mismatch: {scan_path}") + if scan_path in executable_paths: + if result["microsoft_category"] not in {"undetected", "harmless"}: + fail(f"executable result lacks a decisive clean Microsoft verdict: {scan_path}") + if not result["microsoft_engine_version"] or not result["microsoft_engine_update"]: + fail(f"executable result lacks Microsoft version evidence: {scan_path}") + results_by_path[scan_path] = result +if set(results_by_path) != set(association_counts): + fail("VirusTotal result paths differ from association paths") +for association in associations: + result = results_by_path[association["scan_path"]] + if ( + result["sha256"] != association["object_sha256"] + or result["size"] != association["size"] + ): + fail(f"result hash/size differs from association: {association['scan_path']}") + +archive_rows = [row for row in associations if row["association_type"] == "archive"] +member_rows = [row for row in associations if row["association_type"] == "member"] +asset_rows = [row for row in associations if row["association_type"] == "pack_asset"] +if len(archive_rows) != association_meta.get("archives"): + fail("downloadable archive association count is inconsistent") +archive_names = [row["archive"] for row in archive_rows] +if len(set(archive_names)) != len(archive_names): + fail("downloadable archive association is duplicated") +archive_hashes = {row["archive"]: row["archive_sha256"] for row in archive_rows} +if any(row["archive_sha256"] != row["object_sha256"] for row in archive_rows): + fail("downloadable archive SHA is not bound to its scan object") +if any(archive_hashes.get(row["archive"]) != row["archive_sha256"] for row in associations): + fail("member/asset association is not bound to its downloadable archive") + +engine_counts = [int(row["completed_engines"]) for row in results] +minimum = min(engine_counts) +maximum = max(engine_counts) +policy = result_meta.get("min_engines_policy") +if policy is None or minimum < policy: + fail("result engine counts do not satisfy their recorded policy") +engine_range = str(minimum) if minimum == maximum else f"{minimum}–{maximum}" + +section = [ + START, + "## Security Verification", + "", + ( + f"VirusTotal completed **{len(results)} distinct byte objects** covering " + f"**{len(associations)} exact release associations**: " + f"{len(archive_rows)} downloadable archives, {len(member_rows)} archive members, " + f"and {len(asset_rows)} independently extracted UI-pack assets." + ), + "", + ( + f"Every object returned **0 malicious and 0 suspicious** verdicts with " + f"{engine_range} decisive engine results (required minimum: {policy})." + ), + ( + f"Microsoft returned a decisive clean verdict for all " + f"{len(executable_paths)} executable objects." + ), + "", + ( + "Durable public evidence: " + f"[associations]({asset_base}/virustotal-associations.tsv), " + f"[exact scan set]({asset_base}/virustotal-scan-set.tsv), " + f"[per-object results and report links]({asset_base}/virustotal-results.tsv), " + f"[evidence checksums]({asset_base}/virustotal-evidence-checksums.txt)." + ), + "", + "| Downloadable archive | SHA-256 | Engines | VirusTotal |", + "|---|---|---:|---|", +] +for association in sorted(archive_rows, key=lambda row: row["archive"]): + result = results_by_path[association["scan_path"]] + sha256 = result["sha256"] + section.append( + f"| `{association['archive']}` | `{sha256}` | " + f"{result['completed_engines']} | [0 detections]({result['virustotal_url']}) |" + ) +section.append(END) +replacement = "\n".join(section) + +current = current_path.read_text(encoding="utf-8") +start_count = current.count(START) +end_count = current.count(END) +if start_count != end_count or start_count > 1: + fail("existing release notes contain malformed verification markers") +if start_count == 1: + if current.index(START) >= current.index(END): + fail("existing release-note verification markers are reversed") + pattern = re.compile(re.escape(START) + r".*?" + re.escape(END), re.DOTALL) + updated, replacements = pattern.subn(replacement, current, count=1) + if replacements != 1: + fail("existing release-note verification section could not be replaced") +else: + updated = current.rstrip() + ("\n\n" if current.strip() else "") + replacement + "\n" +updated_path.write_text(updated, encoding="utf-8") +PY + gh release edit "$VERSION" \ - --notes-file /tmp/release_notes.md --repo "$GITHUB_REPOSITORY" + --notes-file "$WORK/updated.md" --repo "$GITHUB_REPOSITORY" diff --git a/scripts/ci/check-binary-composition.sh b/scripts/ci/check-binary-composition.sh index 1a0167067a..43864e634d 100755 --- a/scripts/ci/check-binary-composition.sh +++ b/scripts/ci/check-binary-composition.sh @@ -3,12 +3,15 @@ # shipped artifact. # # Microsoft Defender's ML classifier flagged the v0.9.1-rc.1 binaries. The -# hardening pass that followed *removed capability* — an executable stack, an -# in-process updater, test-only environment seams, the embedded HTTP/UI server -# and its process enumerator in the standard build. Each of those regresses -# invisibly: one restored #include, one Makefile source list edit, one revived -# call site, and nothing else in CI notices — the binary just quietly gets its -# malware-shaped surface back and the next release gets flagged again. +# hardening pass that followed removed capability or opaque asset bytes — an +# executable stack, an in-process updater, test-only environment seams, +# embedded integration programs, and the in-image frontend bundle. Each of +# those regresses invisibly: one restored #include, one Makefile source-list +# edit, one revived call site, and nothing else in CI notices — the binary just +# quietly restores capability or opaque bytes that this hardening boundary is +# intended to exclude. Their removal reduces attack surface and makes release +# contents independently inspectable; it does not establish which feature, if +# any, caused an opaque third-party ML verdict. # # This script is the proof that each removal stayed removed. It asserts only # NEGATIVE properties (needle absent), plus one canary string we know ships, @@ -39,16 +42,18 @@ esac # S6: worker/Windows test seams read these env vars. A release binary that # still honours them lets any process on the box steer the indexer's child -# processes and file placement — and CBM_TEST_* in a shipped binary is exactly -# the "debug/injection hooks" shape AV heuristics score on. +# processes and file placement. Their absence is a release-security boundary; +# no claim is made about whether a third-party classifier weighs them. SEAM_NEEDLES=( 'CBM_TEST_WORKER_DESCENDANT_PID_FILE' - 'CBM_TEST_WINDOWS_USER_PATH_RUN_ID' + 'CBM_TEST_CRASH_ON' + 'CBM_TEST_HANG_ON' ) -# The in-process updater (download-and-replace-own-binary) is the single most -# malware-shaped behaviour we ever shipped; updates now run from install.sh -# out-of-process. These four needles cover both halves of what was removed: +# The in-process updater combined network download with replacement of its own +# executable. That dual-use behavior is no longer needed in the daemon; updates +# now run from install.sh out-of-process. These four needles cover both halves +# of what was removed: # the download base URL, the checksum URL built on top of it, the GitHub API # release query of the daemon's background version check, and that request's # Accept header (which survives even if the URL is ever assembled at runtime). @@ -72,8 +77,8 @@ SQLITE_LOADEXT_NEEDLES=( ) # The standard artifact must not contain the UI's HTTP server at all: an -# unauthenticated localhost listener plus a process enumerator is precisely the -# behaviour pair a classifier reads as a backdoor. Needles are split across +# unauthenticated localhost listener plus a process enumerator is unnecessary +# capability in the standard composition. Needles are split across # both UI translation units and are independent of each other, so a refactor of # any single one cannot silently disarm the assertion: # 'HTTP/1.1 %d %s' httpd.c response-status writer @@ -95,8 +100,9 @@ UI_HTTP_NEEDLES=( # The binary used to embed nine complete shebang'd shell scripts, their # PowerShell/.cmd twins and two node:child_process client modules — script -# bodies written to disk at 0755, which is precisely the text surface -# Defender's ML scored Trojan:Script/Wacatac.B!ml. They now ship in +# bodies written to disk at 0755. Keeping those programs outside the native +# image narrows executable surface and lets each component be inspected and +# scanned independently. They now ship in # cbm-integrations.json (the binary embeds only its SHA-256); these needles # prove the removal stayed removed. Source-level enforcement lives in # tests/test_no_embedded_scripts_contract.sh; this is the shipped-artifact @@ -108,6 +114,17 @@ SCRIPT_NEEDLES=( 'node:child_process' ) +# UI frontend bytes belong exclusively in the independently scannable pack. +# These are structural bytes from the pack header and Vite's emitted index, +# selected to avoid generic HTML grammar strings that legitimately occur in the +# language parsers linked into every binary. +UI_ASSET_BYTE_NEEDLES=( + 'CBMUIPK' + '' + '' + '\n", +} +mime_ids = {".html": 1, ".js": 2, ".css": 3} + + +def fail(message: str) -> None: + raise AssertionError(message) + + +def is_ui(name: str) -> bool: + return name.startswith("codebase-memory-mcp-ui-") + + +def make_pack(payloads: Dict[str, bytes]) -> bytes: + entries = sorted(payloads.items()) + paths = [path.encode("ascii") for path, _ in entries] + index_size = len(entries) * 24 + paths_size = sum(len(path) for path in paths) + payload_size = sum(len(data) for _, data in entries) + paths_offset = 80 + index_size + payload_offset = paths_offset + paths_size + total = payload_offset + payload_size + pack = bytearray(total) + struct.pack_into( + "<8sHHIIIQQQQQQQ", + pack, + 0, + b"CBMUIPK\0", + 1, + 80, + 0, + len(entries), + 24, + 80, + index_size, + paths_offset, + paths_size, + payload_offset, + payload_size, + total, + ) + path_cursor = 0 + payload_cursor = 0 + for index, ((path, data), raw_path) in enumerate(zip(entries, paths)): + extension = pathlib.PurePosixPath(path).suffix + struct.pack_into( + " bytes: + data = bytearray(valid_pack) + if kind == "bad_pack_magic": + data[0] = ord("X") + elif kind == "bad_pack_bounds": + struct.pack_into(" Dict[str, object]: + return {"name": name, "kind": "regular", "data": data} + + +def archive_entries(name: str, mutation: Optional[str] = None) -> List[Dict[str, object]]: + marker = name.encode("ascii") + windows = name.endswith(".zip") + binary = "codebase-memory-mcp.exe" if windows else "codebase-memory-mcp" + installer = "install.ps1" if windows else "install.sh" + entries = [ + regular(binary, b"binary bytes from " + marker + b"\n"), + regular("cbm-integrations.json", b'{"schema":1,"integrations":[]}\n'), + regular("LICENSE", b"shared release license\n"), + regular( + installer, + b"Write-Output 'shared installer'\n" + if windows + else b"#!/bin/sh\n# shared installer\n", + ), + regular("THIRD_PARTY_NOTICES.md", b"shared third-party notices\n"), + ] + if is_ui(name): + pack = mutate_pack(mutation) if mutation and mutation.startswith("bad_pack_") else valid_pack + entries.append(regular(f"cbm-ui-{hashlib.sha256(pack).hexdigest()}.pack", pack)) + return entries + + +def write_archive(path: pathlib.Path, entries: List[Dict[str, object]]) -> None: + if path.name.endswith(".tar.gz"): + with tarfile.open(path, "w:gz") as archive: + for entry in entries: + name = str(entry["name"]) + kind = str(entry["kind"]) + info = tarfile.TarInfo(name) + info.mtime = 1 + if kind == "regular": + data = bytes(entry["data"]) + info.mode = 0o755 if name == "codebase-memory-mcp" else 0o644 + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + elif kind == "directory": + info.type = tarfile.DIRTYPE + info.mode = 0o755 + archive.addfile(info) + elif kind == "symlink": + info.type = tarfile.SYMTYPE + info.linkname = str(entry["target"]) + archive.addfile(info) + else: + fail(f"unknown fixture entry kind: {kind}") + return + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for entry in entries: + name = str(entry["name"]) + kind = str(entry["kind"]) + info = zipfile.ZipInfo(name) + info.create_system = 3 + if kind == "regular": + mode = 0o755 if name == "codebase-memory-mcp.exe" else 0o644 + info.external_attr = (stat.S_IFREG | mode) << 16 + archive.writestr(info, bytes(entry["data"])) + elif kind == "directory": + if not name.endswith("/"): + info.filename += "/" + info.external_attr = ((stat.S_IFDIR | 0o755) << 16) | 0x10 + archive.writestr(info, b"") + elif kind == "symlink": + info.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(info, str(entry["target"]).encode("ascii")) + else: + fail(f"unknown fixture entry kind: {kind}") + + +Association = Tuple[str, str, str, str] + + +def build_matrix( + archive_dir: pathlib.Path, + mutation: Optional[str] = None, +) -> Dict[Association, bytes]: + archive_dir.mkdir(parents=True) + expected: Dict[Association, bytes] = {} + targets = { + "bad_hash": "codebase-memory-mcp-ui-linux-amd64.tar.gz", + "bad_pack_magic": "codebase-memory-mcp-ui-linux-amd64.tar.gz", + "bad_pack_bounds": "codebase-memory-mcp-ui-linux-arm64.tar.gz", + "bad_pack_path": "codebase-memory-mcp-ui-darwin-amd64.tar.gz", + "bad_pack_mime": "codebase-memory-mcp-ui-darwin-arm64.tar.gz", + "directory": "codebase-memory-mcp-linux-amd64.tar.gz", + "extra_pack": "codebase-memory-mcp-ui-linux-arm64-portable.tar.gz", + "missing_pack": "codebase-memory-mcp-ui-windows-arm64.zip", + "nested_member": "codebase-memory-mcp-darwin-amd64.tar.gz", + "tar_symlink": "codebase-memory-mcp-linux-arm64.tar.gz", + "tar_duplicate": "codebase-memory-mcp-ui-linux-amd64-portable.tar.gz", + "zip_symlink": "codebase-memory-mcp-windows-amd64.zip", + "zip_directory": "codebase-memory-mcp-windows-arm64.zip", + "zip_duplicate": "codebase-memory-mcp-ui-windows-amd64.zip", + } + target = targets.get(mutation) + for name in archive_names: + if mutation == "missing_archive" and name == "codebase-memory-mcp-linux-amd64.tar.gz": + continue + pack_mutation = mutation if name == target and mutation and mutation.startswith("bad_pack_") else None + entries = archive_entries(name, pack_mutation) + if name == target: + if mutation == "bad_hash": + entries = [entry for entry in entries if not str(entry["name"]).endswith(".pack")] + entries.append(regular(f"cbm-ui-{'0' * 64}.pack", valid_pack)) + elif mutation in ("directory", "zip_directory"): + entries.append({"name": "nested/", "kind": "directory"}) + elif mutation in ("tar_duplicate", "zip_duplicate"): + entries.append(regular("LICENSE", b"second license body\n")) + elif mutation == "extra_pack": + second = make_pack({ + "/assets/other.js": b"console.log('other')\n", + "/index.html": b"other\n", + }) + entries.append(regular(f"cbm-ui-{hashlib.sha256(second).hexdigest()}.pack", second)) + elif mutation == "missing_pack": + entries = [entry for entry in entries if not str(entry["name"]).endswith(".pack")] + elif mutation == "nested_member": + entries.append(regular("nested/payload", b"unexpected nested member\n")) + elif mutation in ("tar_symlink", "zip_symlink"): + installer = "install.ps1" if name.endswith(".zip") else "install.sh" + entries = [entry for entry in entries if entry["name"] != installer] + entries.append({"name": installer, "kind": "symlink", "target": "LICENSE"}) + path = archive_dir / name + write_archive(path, entries) + if mutation is None: + expected[("archive", name, "", "")] = path.read_bytes() + for entry in entries: + if entry["kind"] != "regular": + continue + member = str(entry["name"]) + data = bytes(entry["data"]) + expected[("member", name, member, "")] = data + if member.endswith(".pack"): + for asset_path, payload in asset_payloads.items(): + expected[("pack_asset", name, member, asset_path)] = payload + if mutation == "unexpected_archive": + unexpected = "codebase-memory-mcp-freebsd-amd64.tar.gz" + write_archive(archive_dir / unexpected, archive_entries("codebase-memory-mcp-linux-amd64.tar.gz")) + if mutation == "oversized_archive": + with (archive_dir / "codebase-memory-mcp-linux-amd64.tar.gz").open("r+b") as handle: + handle.truncate(512 * 1024 * 1024 + 1) + return expected + + +def invoke( + archive_dir: pathlib.Path, + output_dir: pathlib.Path, + args: Optional[List[str]] = None, +) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(extractor), str(archive_dir), str(output_dir), *(args or count_args)], + cwd=root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + +def parse_manifest(path: pathlib.Path, marker: str) -> Tuple[Dict[str, int], List[Dict[str, str]]]: + lines = path.read_text(encoding="utf-8").splitlines() + if not lines or lines[0] != f"# {marker}": + fail(f"{path.name} is missing marker {marker}") + metadata: Dict[str, int] = {} + cursor = 1 + while cursor < len(lines) and lines[cursor].startswith("# "): + key, value = lines[cursor][2:].split("=", 1) + metadata[key] = int(value) + cursor += 1 + return metadata, list(csv.DictReader(lines[cursor:], delimiter="\t")) + + +def assert_clean_failure(case_root: pathlib.Path, mutation: str, message: str) -> None: + archive_dir = case_root / "archives" + output_dir = case_root / "scan" + build_matrix(archive_dir, mutation) + result = invoke(archive_dir, output_dir) + if result.returncode == 0 or message not in result.stdout: + fail( + f"{mutation}: extractor must fail with {message!r}; " + f"rc={result.returncode}, output={result.stdout.strip()}" + ) + if output_dir.exists(): + fail(f"{mutation}: rejected archives left a partial atomic scan bundle") + + +if len(archive_names) != 16: + fail("fixture matrix itself no longer describes 16 canonical archives") + +with tempfile.TemporaryDirectory(prefix="cbm-release-scan-contract-") as temporary: + work = pathlib.Path(temporary) + valid = work / "valid" + expected = build_matrix(valid / "archives") + output = valid / "scan" + result = invoke(valid / "archives", output) + if result.returncode != 0: + fail(f"canonical release matrix was rejected: {result.stdout.strip()}") + if {path.name for path in output.iterdir()} != {"objects", "associations.tsv", "scan-set.tsv"}: + fail("scan bundle must publish exactly objects/ plus its two manifests") + + association_metadata, rows = parse_manifest( + output / "associations.tsv", "cbm-release-scan-associations-v2" + ) + expected_pack_assets = 8 * len(asset_payloads) + expected_associations = 16 + 88 + expected_pack_assets + expected_metadata = { + **expected_base_counts, + "pack_assets": expected_pack_assets, + "associations": expected_associations, + "scan_objects": len(set(expected.values())), + } + if association_metadata != expected_metadata: + fail(f"association metadata mismatch: {association_metadata!r}") + if len(rows) != expected_associations: + fail("association manifest does not contain every archive/member/asset association") + actual_associations = { + (row["association_type"], row["archive"], row["member"], row["asset_path"]): row + for row in rows + } + if set(actual_associations) != set(expected): + fail("association manifest keys differ from exact shipped associations") + for association, data in expected.items(): + row = actual_associations[association] + object_path = output / row["scan_path"] + if object_path.read_bytes() != data: + fail(f"scan object changed bytes for association {association!r}") + if row["object_sha256"] != hashlib.sha256(data).hexdigest() or int(row["size"]) != len(data): + fail(f"association digest/size is wrong for {association!r}") + + object_paths = sorted((output / "objects").iterdir()) + if len(object_paths) != len(set(expected.values())): + fail("byte-identical archive members/assets were not deduplicated") + object_bytes = [path.read_bytes() for path in object_paths] + if len(object_bytes) != len(set(object_bytes)): + fail("scan set contains duplicate byte sequences") + license_rows = [row for row in rows if row["member"] == "LICENSE"] + if len(license_rows) != 16 or len({row["scan_path"] for row in license_rows}) != 1: + fail("shared LICENSE bytes must map 16 associations to one scan object") + pack_rows = [row for row in rows if row["kind"] == "pack"] + if len(pack_rows) != 8 or len({row["scan_path"] for row in pack_rows}) != 1: + fail("shared UI pack bytes must map eight associations to one scan object") + script_rows = [row for row in rows if row["asset_path"] == "/assets/app.js"] + if len(script_rows) != 8 or len({row["scan_path"] for row in script_rows}) != 1: + fail("each pack's JavaScript association must reach one exact deduplicated script object") + + scan_metadata, scan_rows = parse_manifest(output / "scan-set.tsv", "cbm-release-scan-set-v1") + if scan_metadata != { + "scan_objects": expected_metadata["scan_objects"], + "associations": expected_associations, + }: + fail(f"scan-set metadata mismatch: {scan_metadata!r}") + if len(scan_rows) != len(object_paths): + fail("scan-set row count differs from staged distinct objects") + scan_paths = {row["scan_path"] for row in scan_rows} + if scan_paths != {f"objects/{path.name}" for path in object_paths}: + fail("scan-set paths differ from staged object paths") + if sum(int(row["association_count"]) for row in scan_rows) != expected_associations: + fail("scan-set association counts do not cover the association manifest") + for row in scan_rows: + path = output / row["scan_path"] + data = path.read_bytes() + if row["sha256"] != hashlib.sha256(data).hexdigest() or int(row["size"]) != len(data): + fail(f"scan-set digest/size does not bind {row['scan_path']}") + + failure_cases = { + "bad_hash": "UI pack digest does not match its filename", + "bad_pack_magic": "CBMUIPK header/offset contract failed", + "bad_pack_bounds": "CBMUIPK header/offset contract failed", + "bad_pack_path": "CBMUIPK asset path is invalid", + "bad_pack_mime": "CBMUIPK MIME/path contract failed", + "directory": "non-regular archive member", + "extra_pack": "exactly one UI pack", + "missing_archive": "expected exact canonical 16", + "missing_pack": "exactly one UI pack", + "nested_member": "unexpected archive member", + "oversized_archive": "archive exceeds 536870912 byte ceiling", + "tar_symlink": "non-regular archive member", + "tar_duplicate": "duplicate archive member", + "unexpected_archive": "expected exact canonical 16", + "zip_directory": "non-regular archive member", + "zip_duplicate": "duplicate archive member", + "zip_symlink": "non-regular archive member", + } + for mutation, message in failure_cases.items(): + case_root = work / mutation + case_root.mkdir() + assert_clean_failure(case_root, mutation, message) + + wrong_count = work / "wrong-count" + wrong_count.mkdir() + build_matrix(wrong_count / "archives") + wrong_args = [arg for arg in count_args if not arg.startswith("--expect-packs=")] + wrong_args.append("--expect-packs=9") + wrong_output = wrong_count / "scan" + wrong = invoke(wrong_count / "archives", wrong_output, wrong_args) + if wrong.returncode == 0 or "packs expected 9, got 8" not in wrong.stdout or wrong_output.exists(): + fail(f"explicit expected-count mismatch did not fail atomically: {wrong.stdout.strip()}") + +source = extractor.read_text(encoding="utf-8") +if "files_equal(candidate, existing.path)" not in source: + fail("deduplication no longer exact-compares bytes after SHA-256+size grouping") +if source.count("archive_object.path,") < 2: + fail("members are not parsed from the same retained bytes staged as the archive object") +for ceiling in ("MAX_ARCHIVE_BYTES", "MAX_MEMBER_BYTES", "MAX_TOTAL_MEMBER_BYTES"): + if ceiling not in source: + fail(f"extractor lost required size ceiling {ceiling}") + +release = (root / ".github/workflows/release.yml").read_text(encoding="utf-8") +dry_run = (root / ".github/workflows/dry-run.yml").read_text(encoding="utf-8") +for workflow_name, workflow in (("release", release), ("dry-run", dry_run)): + for argument in count_args: + if argument not in workflow: + fail(f"{workflow_name} workflow does not assert {argument}") + for required in ( + "files: binaries/objects/*", + "request_rate: 4", + "VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv", + "VT_ASSOCIATIONS: binaries/associations.tsv", + "VT_RESULTS_PATH: binaries/vt-results.tsv", + "MIN_ENGINES: 50", + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + "virustotal-evidence-", + ): + if required not in workflow: + fail(f"{workflow_name} workflow is missing exact scan/gate contract: {required}") + lowered = workflow.lower() + for forbidden in ("false-positive", "false positive", "submission", "reputation"): + if forbidden in lowered: + fail(f"{workflow_name} workflow prescribes forbidden scan-cleaning policy: {forbidden}") + if "naturally clean" not in lowered: + fail(f"{workflow_name} workflow must state the naturally-clean baseline") + +notes = (root / "scripts/ci/append-vt-notes.sh").read_text(encoding="utf-8") +for required in ( + "cbm-security-verification:start", + "cbm-security-verification:end", + "binaries/associations.tsv", + "binaries/vt-results.tsv", +): + if required not in notes: + fail(f"release-note updater is missing {required}") +if "70+" in notes or "...`" in notes: + fail("release notes must use measured engine counts and full SHA-256 values") + +print("release archive extractor contract: OK") +PY diff --git a/tests/test_security.c b/tests/test_security.c index 8f2731984a..fc81f644be 100644 --- a/tests/test_security.c +++ b/tests/test_security.c @@ -75,7 +75,8 @@ TEST(vendored_integrity_manifest_is_relocatable_and_fail_closed) { if (sscanf(line, "%64s %3899s", hash, path) != 2) continue; ASSERT_EQ(strlen(hash), 64U); - ASSERT(strncmp(path, "vendored/", strlen("vendored/")) == 0); + ASSERT(strncmp(path, "vendored/", strlen("vendored/")) == 0 || + strncmp(path, "internal/cbm/vendored/", strlen("internal/cbm/vendored/")) == 0); entries++; } fclose(checksums); diff --git a/tests/test_smoke_fixture_contract.sh b/tests/test_smoke_fixture_contract.sh index 111b5c7cd7..177d386211 100755 --- a/tests/test_smoke_fixture_contract.sh +++ b/tests/test_smoke_fixture_contract.sh @@ -8,10 +8,13 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd)" python3 - "$ROOT" <<'PY' from __future__ import annotations +import hashlib +import os import pathlib import re import subprocess import sys +import tarfile import tempfile import time import urllib.request @@ -44,9 +47,29 @@ local_ci = read("test-infrastructure/run.sh") windows_vm = read("test-infrastructure/vm/win.sh") windows_vm_runner = read("test-infrastructure/vm/vm-run-tests.sh") pr_workflow = read(".github/workflows/pr.yml") +release_workflow = read(".github/workflows/release.yml") +release_extractor = read("scripts/ci/extract-release-archives.sh") test_driver = read("scripts/test.sh") cli_source = read("src/cli/cli.c") helper_source = read(helper_relative) +unix_installer = read("install.sh") +windows_installer = read("install.ps1") + +# Keep the security-boundary archive checks runnable as a focused, network-free +# contract while still making them part of the canonical smoke-fixture gate. +release_archive_contract = subprocess.run( + ["bash", str(root / "tests/test_release_archive_extractor_contract.sh")], + cwd=root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, +) +require( + release_archive_contract.returncode == 0, + "release archive extractor contract failed: " + + release_archive_contract.stdout.strip(), +) # The server owns the ephemeral bind. A parent-side socket probe followed by # python -m http.server would reintroduce the close/rebind race this guards. @@ -134,18 +157,23 @@ for relative, source in ( f"{relative} must pin isolated {variable}", ) -# Unix fixtures mirror the release archive surface and Linux update aliases. +# Unix fixtures mirror the exact release variant surface. for name in ("LICENSE", "install.sh", "THIRD_PARTY_NOTICES.md"): require(name in smoke_local, f"smoke-local.sh archive must include {name}") require( "codebase-memory-mcp${SUFFIX}-${OS}-${ARCH}.tar.gz" in smoke_local - and "codebase-memory-mcp-${OS}-${ARCH}.tar.gz" in smoke_local, - "smoke-local.sh must create the selected variant and standard alias", + and "UI_PACK_NAME" in smoke_local + and "^cbm-ui-[0-9a-f]{64}\\.pack$" in smoke_local, + "smoke-local.sh must create the selected exact variant with a hash-shaped UI pack", ) require( "codebase-memory-mcp${SUFFIX}-${OS}-${ARCH}-portable.tar.gz" in smoke_local - and "codebase-memory-mcp-${OS}-${ARCH}-portable.tar.gz" in smoke_local, - "smoke-local.sh must create Linux portable selected-variant and standard aliases", + and 'ARCHIVE_MEMBERS+=("$UI_PACK_NAME")' in smoke_local, + "smoke-local.sh must create the Linux portable selected variant with the same member set", +) +require( + '"$FIXTURE_DIR/codebase-memory-mcp-${OS}-${ARCH}.tar.gz"' not in smoke_local, + "smoke-local.sh must not disguise a six-member UI archive under a standard name", ) require( 'CBM_CACHE_DIR="$WORK_DIR/cache"' in smoke_local @@ -163,9 +191,9 @@ require( "fixture checksums must name exact artifact basenames, never ./-prefixed paths", ) -# Native Windows packages and serves the exact four-file release bundle (ONE -# binary, like every other platform), then runs the full smoke from a protected -# profile-rooted directory/cache. +# Native Windows packages and serves the exact five-file standard bundle or +# six-file UI bundle (one native binary), then runs the full smoke from a +# protected profile-rooted directory/cache. for name in ( "codebase-memory-mcp.exe", "LICENSE", @@ -178,6 +206,16 @@ require( "vm-smoke.sh must not stage a Windows launcher/payload pair", ) require("checksums.txt" in vm_smoke, "vm-smoke.sh must generate checksums.txt") +require( + "UI_PACK_NAME" in vm_smoke + and "^cbm-ui-[0-9a-f]{64}\\.pack$" in vm_smoke + and 'ARCHIVE_MEMBERS+=("$UI_PACK_NAME")' in vm_smoke, + "vm-smoke.sh must stage exactly one content-addressed pack only for UI", +) +require( + '"codebase-memory-mcp-windows-${SMOKE_ARCH}.zip"' not in vm_smoke, + "vm-smoke.sh must not disguise a six-member UI zip under a standard name", +) require( "SMOKE_DOWNLOAD_URL=" in vm_smoke and "SMOKE_UPDATE_FIXTURE_DIR=" in vm_smoke @@ -262,7 +300,7 @@ for service in ("smoke-windows:",): require( "wine64 ./build/win-cross/codebase-memory-mcp.exe --version" in compose and "wine64 cmd /c build/win-cross/codebase-memory-mcp.exe --version" in compose, - "docker-compose Windows cross-smoke must execute the single binary through Wine and through a " + "docker-compose Windows cross-smoke must execute the runtime set's one executable through Wine and through a " "Wine Windows parent", ) require( @@ -305,6 +343,92 @@ require( "PR Windows smoke must call vm-smoke.sh with SMOKE_ARCH=amd64", ) smoke_test = read("scripts/smoke-test.sh") +require( + "SMOKE_UI_PACK_CANDIDATES" in smoke_test + and 'cp "$SMOKE_UI_PACK" "$(dirname "$destination")/$(basename "$SMOKE_UI_PACK")"' + in smoke_test, + "smoke-test.sh must stage the UI sidecar with every hand-copied binary", +) +require( + 'UNINSTALL_BINARY="$SELF_PATH"' in smoke_test + and '"$CLEAN_UNINSTALLER" uninstall -y -n' in smoke_test + and '"$DBL_UNINSTALLER" uninstall -y -n' in smoke_test + and '"$DBL_RETRY_UNINSTALLER" uninstall -y -n' in smoke_test, + "smoke uninstall cases must run disposable fixture executables, never delete the " + "build input needed by later phases", +) +require( + 'exec 9< <(sleep 300)' in smoke_test + and 'UI_STDIN_PID=$!' in smoke_test + and smoke_test.count("stop_ui_probe") >= 4 + and 'sleep 300 | "$BINARY"' not in smoke_test, + "the UI readiness probe must own and reap its stdin keeper instead of waiting on a " + "background pipeline", +) +require( + '"$REPO_ROOT/install.sh" "$DL_VARIANT_ARG"' in smoke_test + and '"$WIN_SCRIPT" "$DL_VARIANT_ARG"' in smoke_test + and "install.ps1 did not publish exactly one UI asset pack" in smoke_test, + "Phase 13 must call each direct installer with the downloaded variant and verify its pack", +) +require( + all( + needle in unix_installer + for needle in ( + "ARCHIVE_MEMBER_COUNT", + "EXPECTED_MEMBER_COUNT=5", + "EXPECTED_MEMBER_COUNT=6", + "^cbm-ui-[0-9a-f]{64}\\.pack$", + "release archive contains unexpected member", + ) + ), + "install.sh must validate the exact five-member standard or six-member UI archive", +) +unix_binary_publish = unix_installer.find('"$DLBIN" install "${INSTALL_ARGS[@]}"') +require( + unix_binary_publish >= 0 + and "publish_required_sidecar" not in unix_installer + and 'for stale_pack in "$INSTALL_DIR"/cbm-ui-*.pack' not in unix_installer, + "install.sh must delegate sidecar publication and stale-pack GC to the candidate's " + "guarded native install", +) +require( + "GetRandomFileName" in windows_installer + and "[System.IO.FileMode]::CreateNew" in windows_installer + and windows_installer.count("New-CbmExclusiveSiblingTemp") == 2 + and '$Destination.new-$PID' not in windows_installer + and '$InstallerDest.new' not in windows_installer, + "install.ps1 must reserve an unpredictable sibling temp exclusively for the updater only", +) + +cli_source = read("src/cli/cli.c") +install_start = cli_source.find("static int cli_install_activate(void *opaque)") +install_end = cli_source.find("int cbm_cmd_install(", install_start) +install_body = cli_source[install_start:install_end] if 0 <= install_start < install_end else "" +runtime_stage = install_body.find("cbm_integration_assets_stage_install") +binary_commit = install_body.find("cli_activation_transaction_commit_validated") +runtime_gc = install_body.find("cli_gc_stale_ui_asset_packs") +runtime_finalize = install_body.find("cli_install_runtime_finalize") +require( + 0 <= runtime_stage < binary_commit < runtime_gc < runtime_finalize, + "native install must stage sidecars, activate the binary, GC verified stale packs, and " + "finalize while the activation callback still owns the guard", +) +require( + "scripts/ci/extract-release-archives.sh assets binaries" in release_workflow, + "release verification must use the canonical archive extractor", +) +require( + "cbm-release-scan-associations-v2" in release_extractor + and "cbm-release-scan-set-v1" in release_extractor + and "files_equal(candidate, existing.path)" in release_extractor + and "parse_pack_assets" in release_extractor + and "PACK_NAME.fullmatch" in release_extractor + and "non-regular archive member" in release_extractor + and "duplicate archive member" in release_extractor, + "release extraction must validate every exact archive/member/UI-asset association " + "and retain one exact-compared scan object per distinct byte sequence", +) require( "MSYS2_ARG_CONV_EXCL='*'" in smoke_test and 'powershell.exe -NoProfile -ExecutionPolicy Bypass -File' in smoke_test @@ -523,6 +647,218 @@ if helper.is_file(): dns_process.kill() dns_process.wait(timeout=3) +# POSIX installer lifecycle injection. Preserve the wrapper's PID across exec, +# precreate the two formerly predictable temp paths as foreign symlinks, then +# run the real checksum-authenticated installer. The script no longer publishes +# runtime sidecars at all, and its updater publisher never touches the foreign +# predictable sibling. +if os.name != "nt" and helper.is_file(): + with tempfile.TemporaryDirectory(prefix="cbm-installer-temp-contract-") as install_temp: + install_root = pathlib.Path(install_temp) + fixture = install_root / "fixture" + stage = install_root / "stage" + fake_bin = install_root / "fake-bin" + install_dir = install_root / "installed" + fake_home = install_root / "home" + fixture.mkdir() + stage.mkdir() + fake_bin.mkdir() + install_dir.mkdir() + fake_home.mkdir() + + candidate = stage / "codebase-memory-mcp" + candidate.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "case \"${1:-}\" in\n" + " --version) echo 'codebase-memory-mcp fixture';;\n" + " install)\n" + " install_dir=''\n" + " for arg in \"$@\"; do\n" + " case \"$arg\" in --dir=*) install_dir=${arg#--dir=};; esac\n" + " done\n" + " test -n \"$install_dir\"\n" + " cp \"$0\" \"$install_dir/codebase-memory-mcp\"\n" + " chmod 755 \"$install_dir/codebase-memory-mcp\";;\n" + " *) exit 2;;\n" + "esac\n", + encoding="utf-8", + ) + candidate.chmod(0o755) + (stage / "cbm-integrations.json").write_bytes(b"{}\n") + (stage / "LICENSE").write_bytes(b"fixture license\n") + (stage / "THIRD_PARTY_NOTICES.md").write_bytes(b"fixture notices\n") + + archive_name = "codebase-memory-mcp-linux-amd64-portable.tar.gz" + archive_path = fixture / archive_name + with tarfile.open(archive_path, "w:gz") as archive: + for name in ( + "codebase-memory-mcp", + "cbm-integrations.json", + "LICENSE", + "THIRD_PARTY_NOTICES.md", + ): + archive.add(stage / name, arcname=name) + archive.add(root / "install.sh", arcname="install.sh") + archive_digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() + (fixture / "checksums.txt").write_text( + f"{archive_digest} {archive_name}\n", encoding="ascii" + ) + + fake_uname = fake_bin / "uname" + fake_uname.write_text( + "#!/bin/sh\n" + "case \"${1:-}\" in\n" + " -s) echo Linux;;\n" + " -m) echo x86_64;;\n" + " *) echo Linux;;\n" + "esac\n", + encoding="utf-8", + ) + fake_uname.chmod(0o755) + + foreign_sidecar = install_root / "foreign-sidecar" + foreign_updater = install_root / "foreign-updater" + foreign_sidecar.write_bytes(b"foreign sidecar sentinel\n") + foreign_updater.write_bytes(b"foreign updater sentinel\n") + wrapper_pid_file = install_root / "wrapper-pid" + wrapper = install_root / "run-installer.sh" + wrapper.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "install_dir=$1\n" + "foreign_sidecar=$2\n" + "foreign_updater=$3\n" + "pid_file=$4\n" + "installer=$5\n" + "ln -s \"$foreign_sidecar\" \"$install_dir/.cbm-integrations.json.$$\"\n" + "ln -s \"$foreign_updater\" \"$install_dir/.install.sh.$$\"\n" + "printf '%s\\n' \"$$\" > \"$pid_file\"\n" + "exec bash \"$installer\" --standard \"--dir=$install_dir\" --skip-config\n", + encoding="utf-8", + ) + wrapper.chmod(0o755) + + port_file = install_root / "port" + server_log = install_root / "server.log" + with server_log.open("wb") as log_handle: + fixture_server = subprocess.Popen( + [ + sys.executable, + str(helper), + "--directory", + str(fixture), + "--port-file", + str(port_file), + ], + stdout=log_handle, + stderr=subprocess.STDOUT, + ) + try: + port = 0 + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if port_file.is_file(): + try: + port_text = port_file.read_text(encoding="ascii").strip() + except OSError: + port_text = "" + if port_text: + port = int(port_text) + break + if fixture_server.poll() is not None: + break + time.sleep(0.02) + require(port > 0, "installer temp-path fixture server did not publish a port") + if port > 0: + environment = os.environ.copy() + environment["HOME"] = str(fake_home) + environment["PATH"] = ( + str(fake_bin) + os.pathsep + environment.get("PATH", "") + ) + environment["CBM_DOWNLOAD_URL"] = f"http://127.0.0.1:{port}" + installed = subprocess.run( + [ + str(wrapper), + str(install_dir), + str(foreign_sidecar), + str(foreign_updater), + str(wrapper_pid_file), + str(root / "install.sh"), + ], + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=60, + check=False, + ) + require( + installed.returncode == 0, + f"installer temp-path fixture failed to install: {installed.stdout.strip()}", + ) + if wrapper_pid_file.is_file(): + wrapper_pid = wrapper_pid_file.read_text(encoding="ascii").strip() + predictable_sidecar = ( + install_dir / f".cbm-integrations.json.{wrapper_pid}" + ) + predictable_updater = install_dir / f".install.sh.{wrapper_pid}" + require( + predictable_sidecar.is_symlink() + and foreign_sidecar.read_bytes() + == b"foreign sidecar sentinel\n", + "install.sh deleted or followed a foreign predictable sidecar temp", + ) + require( + predictable_updater.is_symlink() + and foreign_updater.read_bytes() + == b"foreign updater sentinel\n", + "install.sh deleted or followed a foreign predictable updater temp", + ) + else: + failures.append("installer temp-path wrapper did not publish its PID") + finally: + fixture_server.terminate() + try: + fixture_server.wait(timeout=3) + except subprocess.TimeoutExpired: + fixture_server.kill() + fixture_server.wait(timeout=3) + +# The complete canonical-matrix extraction, pack parsing, association retention, +# byte-exact deduplication and fail-closed malformed-pack cases run above via +# test_release_archive_extractor_contract.sh. Keep the packaging-side digest +# check here because it is a separate boundary before an archive exists. +with tempfile.TemporaryDirectory(prefix="cbm-release-extract-") as extract_temp: + extract_root = pathlib.Path(extract_temp) + bad_pack_name = f"cbm-ui-{'0' * 64}.pack" + bad_pack_bytes = b"content does not match the zero digest\n" + # Packaging checks the content address before it touches the product binary. + # No binary is staged deliberately: the digest error must win over the + # later missing-binary guard or this test would accept an incidental red. + package_build = extract_root / "package-build" + package_output = extract_root / "package-output" + package_build.mkdir() + package_output.mkdir() + (package_build / bad_pack_name).write_bytes(bad_pack_bytes) + package_environment = os.environ.copy() + package_environment["BUILD_DIR"] = str(package_build) + packaged = subprocess.run( + ["bash", str(root / "scripts/package-release.sh"), "linux", "amd64", + "--variant", "ui", "--out-dir", str(package_output)], + cwd=root, + env=package_environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + require( + packaged.returncode != 0 + and "UI pack digest does not match its filename" in packaged.stdout, + "release packager accepted a UI pack whose content did not match its filename", + ) + if failures: print("smoke fixture contract: FAIL", file=sys.stderr) for failure in failures: diff --git a/tests/test_ui.c b/tests/test_ui.c index a403dbbbec..ba22ee5e60 100644 --- a/tests/test_ui.c +++ b/tests/test_ui.c @@ -1,14 +1,15 @@ /* * test_ui.c — Tests for the graph visualization UI module. * - * Covers: config persistence, embedded asset lookup, layout engine. + * Covers: config persistence, external asset-pack validation, layout engine. */ #include "../src/foundation/compat.h" #include "../src/foundation/compat_fs.h" #include "test_framework.h" #include "test_helpers.h" +#include "foundation/sha256.h" +#include "ui/asset_pack.h" #include "ui/config.h" -#include "ui/embedded_assets.h" #include "ui/layout3d.h" #include "store/store.h" #ifdef _WIN32 @@ -124,9 +125,9 @@ TEST(config_save_atomically_replaces_a_complete_generation) { char old_bytes[512] = {0}; #ifdef _WIN32 DWORD old_length = 0; - bool old_read = ReadFile(old_handle, old_bytes, (DWORD)sizeof(old_bytes) - 1U, &old_length, - NULL) != 0 && - old_length > 0; + bool old_read = + ReadFile(old_handle, old_bytes, (DWORD)sizeof(old_bytes) - 1U, &old_length, NULL) != 0 && + old_length > 0; bool old_closed = CloseHandle(old_handle) != 0; #else size_t old_length = fread(old_bytes, 1, sizeof(old_bytes) - 1, old_handle); @@ -259,18 +260,545 @@ TEST(config_missing_fields) { PASS(); } -/* ── Embedded asset tests ─────────────────────────────────────── */ +/* ── External UI asset-pack tests ────────────────────────────── */ -TEST(embedded_lookup_not_found) { - /* With stub, everything should return NULL */ - const cbm_embedded_file_t *f = cbm_embedded_lookup("/nonexistent"); - ASSERT_NULL(f); +enum { + TEST_UI_PACK_HEADER_BYTES = 80, + TEST_UI_PACK_ENTRY_BYTES = 24, +}; + +static void ui_test_write_u16(unsigned char *bytes, uint16_t value) { + bytes[0] = (unsigned char)value; + bytes[1] = (unsigned char)(value >> 8U); +} + +static void ui_test_write_u32(unsigned char *bytes, uint32_t value) { + for (size_t index = 0U; index < sizeof(value); index++) { + bytes[index] = (unsigned char)(value >> (index * 8U)); + } +} + +static void ui_test_write_u64(unsigned char *bytes, uint64_t value) { + for (size_t index = 0U; index < sizeof(value); index++) { + bytes[index] = (unsigned char)(value >> (index * 8U)); + } +} + +static unsigned char *ui_test_pack(size_t *length_out, bool malformed_index) { + static const char asset_path[] = "/assets/app.js"; + static const char index_path[] = "/index.html"; + static const unsigned char asset_data[] = "console.log(1);"; + static const unsigned char index_data[] = "

CBM

"; + size_t paths_length = sizeof(asset_path) - 1U + sizeof(index_path) - 1U; + size_t payload_length = sizeof(asset_data) - 1U + sizeof(index_data) - 1U; + size_t index_length = TEST_UI_PACK_ENTRY_BYTES * 2U; + size_t paths_offset = TEST_UI_PACK_HEADER_BYTES + index_length; + size_t payload_offset = paths_offset + paths_length; + size_t total = payload_offset + payload_length; + unsigned char *pack = calloc(total, 1U); + if (!pack) { + return NULL; + } + pack[0] = 'C'; + pack[1] = 'B'; + pack[2] = 'M'; + pack[3] = 'U'; + pack[4] = 'I'; + pack[5] = 'P'; + pack[6] = 'K'; + ui_test_write_u16(pack + 8U, 1U); + ui_test_write_u16(pack + 10U, TEST_UI_PACK_HEADER_BYTES); + ui_test_write_u32(pack + 16U, 2U); + ui_test_write_u32(pack + 20U, TEST_UI_PACK_ENTRY_BYTES); + ui_test_write_u64(pack + 24U, TEST_UI_PACK_HEADER_BYTES); + ui_test_write_u64(pack + 32U, index_length); + ui_test_write_u64(pack + 40U, paths_offset); + ui_test_write_u64(pack + 48U, paths_length); + ui_test_write_u64(pack + 56U, payload_offset); + ui_test_write_u64(pack + 64U, payload_length); + ui_test_write_u64(pack + 72U, total); + + unsigned char *asset_entry = pack + TEST_UI_PACK_HEADER_BYTES; + ui_test_write_u32(asset_entry, malformed_index ? 1U : 0U); + ui_test_write_u16(asset_entry + 4U, sizeof(asset_path) - 1U); + asset_entry[6U] = 2U; + asset_entry[7U] = CBM_UI_ASSET_IMMUTABLE; + ui_test_write_u64(asset_entry + 8U, 0U); + ui_test_write_u64(asset_entry + 16U, sizeof(asset_data) - 1U); + + unsigned char *index_entry = asset_entry + TEST_UI_PACK_ENTRY_BYTES; + ui_test_write_u32(index_entry, sizeof(asset_path) - 1U); + ui_test_write_u16(index_entry + 4U, sizeof(index_path) - 1U); + index_entry[6U] = 1U; + index_entry[7U] = CBM_UI_ASSET_REVALIDATE; + ui_test_write_u64(index_entry + 8U, sizeof(asset_data) - 1U); + ui_test_write_u64(index_entry + 16U, sizeof(index_data) - 1U); + + memcpy(pack + paths_offset, asset_path, sizeof(asset_path) - 1U); + memcpy(pack + paths_offset + sizeof(asset_path) - 1U, index_path, sizeof(index_path) - 1U); + memcpy(pack + payload_offset, asset_data, sizeof(asset_data) - 1U); + memcpy(pack + payload_offset + sizeof(asset_data) - 1U, index_data, sizeof(index_data) - 1U); + *length_out = total; + return pack; +} + +static bool ui_test_publish_pack(const char *directory, unsigned char *pack, size_t length, + char path_out[1024]) { + char hash[CBM_SHA256_HEX_LEN + 1U]; + cbm_sha256_hex(pack, length, hash); + char name[128]; + int name_length = snprintf(name, sizeof(name), "cbm-ui-%s.pack", hash); + int path_length = snprintf(path_out, 1024U, "%s/%s", directory, name); + if (name_length <= 0 || (size_t)name_length >= sizeof(name) || path_length <= 0 || + path_length >= 1024) { + return false; + } + FILE *file = fopen(path_out, "wb"); + bool written = file && fwrite(pack, 1U, length, file) == length; + if (file && fclose(file) != 0) { + written = false; + } + if (written) { + cbm_ui_assets_set_manifest_for_testing(name, hash, length); + } + return written; +} + +static void ui_test_restore_assets_env(char *old_value) { + if (old_value) { + (void)cbm_setenv("CBM_UI_ASSETS_DIR", old_value, 1); + free(old_value); + } else { + (void)cbm_unsetenv("CBM_UI_ASSETS_DIR"); + } +} + +TEST(ui_asset_pack_validates_before_publication) { + char directory[] = "/tmp/cbm_ui_pack_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(directory)); + size_t length = 0U; + unsigned char *pack = ui_test_pack(&length, false); + ASSERT_NOT_NULL(pack); + char pack_path[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(directory, pack, length, pack_path)); + free(pack); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", directory, 1), 0); + + char error[512]; + ASSERT_TRUE(cbm_ui_assets_supported()); + ASSERT_EQ(cbm_ui_assets_state(), CBM_UI_ASSETS_COLD); + ASSERT_TRUE(cbm_ui_assets_warm(NULL, error, sizeof(error))); + ASSERT_EQ(cbm_ui_assets_state(), CBM_UI_ASSETS_READY); + const cbm_ui_asset_t *asset = cbm_ui_asset_lookup("/assets/app.js"); + ASSERT_NOT_NULL(asset); + ASSERT_EQ(asset->cache, CBM_UI_ASSET_IMMUTABLE); + ASSERT_STR_EQ(asset->content_type, "application/javascript; charset=utf-8"); + ASSERT_EQ(asset->size, sizeof("console.log(1);") - 1U); + ASSERT_NULL(cbm_ui_asset_lookup("/api/processes")); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(pack_path); + (void)cbm_rmdir(directory); + PASS(); +} + +TEST(ui_asset_pack_wrong_hash_fails_closed) { + char directory[] = "/tmp/cbm_ui_hash_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(directory)); + size_t length = 0U; + unsigned char *pack = ui_test_pack(&length, false); + ASSERT_NOT_NULL(pack); + char pack_path[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(directory, pack, length, pack_path)); + pack[length - 1U] ^= 1U; + FILE *file = fopen(pack_path, "wb"); + ASSERT_NOT_NULL(file); + ASSERT_EQ(fwrite(pack, 1U, length, file), length); + ASSERT_EQ(fclose(file), 0); + free(pack); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", directory, 1), 0); + + char error[512]; + ASSERT_FALSE(cbm_ui_assets_warm(NULL, error, sizeof(error))); + ASSERT_EQ(cbm_ui_assets_state(), CBM_UI_ASSETS_FAILED); + ASSERT_NULL(cbm_ui_asset_lookup("/index.html")); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(pack_path); + (void)cbm_rmdir(directory); + PASS(); +} + +TEST(ui_asset_pack_structural_corruption_fails_after_valid_hash) { + char directory[] = "/tmp/cbm_ui_format_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(directory)); + size_t length = 0U; + unsigned char *pack = ui_test_pack(&length, true); + ASSERT_NOT_NULL(pack); + char pack_path[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(directory, pack, length, pack_path)); + free(pack); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", directory, 1), 0); + + char error[512]; + ASSERT_FALSE(cbm_ui_assets_warm(NULL, error, sizeof(error))); + ASSERT_EQ(cbm_ui_assets_state(), CBM_UI_ASSETS_FAILED); + ASSERT_NULL(cbm_ui_asset_lookup("/index.html")); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(pack_path); + (void)cbm_rmdir(directory); + PASS(); +} + +/* A daemon can be asked to stop immediately after creating the warmup thread, + * before that worker enters cbm_ui_assets_warm(). The cancellation must latch + * across that scheduling window instead of being cleared by worker startup. */ +TEST(ui_asset_pack_prestart_cancellation_is_not_lost) { + char directory[] = "/tmp/cbm_ui_cancel_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(directory)); + size_t length = 0U; + unsigned char *pack = ui_test_pack(&length, false); + ASSERT_NOT_NULL(pack); + char pack_path[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(directory, pack, length, pack_path)); + free(pack); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", directory, 1), 0); + + cbm_ui_assets_request_cancel(); + char error[512]; + bool warmed = cbm_ui_assets_warm(NULL, error, sizeof(error)); + cbm_ui_assets_state_t state = cbm_ui_assets_state(); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(pack_path); + (void)cbm_rmdir(directory); + + ASSERT_FALSE(warmed); + ASSERT_EQ(state, CBM_UI_ASSETS_CANCELLED); + PASS(); +} + +TEST(ui_asset_pack_install_never_follows_a_predictable_temp_symlink) { +#ifdef _WIN32 + SKIP_PLATFORM( + "Windows reparse-point creation requires privileges; installer contracts cover it"); +#else + static const char sentinel_content[] = "foreign sentinel must survive\n"; + char source_directory[] = "/tmp/cbm_ui_install_source_XXXXXX"; + char install_directory[] = "/tmp/cbm_ui_install_target_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(source_directory)); + ASSERT_NOT_NULL(cbm_mkdtemp(install_directory)); + + size_t pack_length = 0U; + unsigned char *pack = ui_test_pack(&pack_length, false); + ASSERT_NOT_NULL(pack); + char source_pack[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(source_directory, pack, pack_length, source_pack)); + free(pack); + + const char *pack_name = strrchr(source_pack, '/'); + ASSERT_NOT_NULL(pack_name); + pack_name++; + char installed_pack[1024]; + char old_temporary[1024]; + char sentinel_path[1024]; + ASSERT_LT( + snprintf(installed_pack, sizeof(installed_pack), "%s/%s", install_directory, pack_name), + (int)sizeof(installed_pack)); + ASSERT_LT(snprintf(old_temporary, sizeof(old_temporary), "%s.tmp", installed_pack), + (int)sizeof(old_temporary)); + ASSERT_LT(snprintf(sentinel_path, sizeof(sentinel_path), "%s/foreign.txt", install_directory), + (int)sizeof(sentinel_path)); + ASSERT_EQ(th_write_file(sentinel_path, sentinel_content), 0); + ASSERT_EQ(symlink(sentinel_path, old_temporary), 0); + + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", source_directory, 1), 0); + char error[512] = {0}; + bool installed = cbm_ui_assets_install(install_directory, false, error, sizeof(error)); + + char sentinel_after[sizeof(sentinel_content)] = {0}; + FILE *sentinel_file = cbm_fopen(sentinel_path, "rb"); + size_t sentinel_length = + sentinel_file ? fread(sentinel_after, 1U, sizeof(sentinel_after), sentinel_file) : 0U; + bool sentinel_closed = !sentinel_file || fclose(sentinel_file) == 0; + struct stat old_temp_info; + bool old_temp_is_still_symlink = + lstat(old_temporary, &old_temp_info) == 0 && S_ISLNK(old_temp_info.st_mode); + cbm_path_info_t installed_info; + bool installed_is_regular = cbm_path_info_utf8(installed_pack, &installed_info) == 0 && + installed_info.is_regular && !installed_info.is_symlink; + bool removed = cbm_ui_assets_remove(install_directory, false, error, sizeof(error)); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(old_temporary); + (void)cbm_unlink(installed_pack); + (void)cbm_unlink(sentinel_path); + (void)cbm_unlink(source_pack); + (void)cbm_rmdir(install_directory); + (void)cbm_rmdir(source_directory); + + ASSERT_TRUE(installed); + ASSERT_TRUE(sentinel_closed); + ASSERT_EQ(sentinel_length, sizeof(sentinel_content) - 1U); + ASSERT_MEM_EQ(sentinel_after, sentinel_content, sizeof(sentinel_content) - 1U); + ASSERT_TRUE(old_temp_is_still_symlink); + ASSERT_TRUE(installed_is_regular); + ASSERT_TRUE(removed); + PASS(); +#endif +} + +TEST(ui_asset_pack_install_remove_is_hash_bound_and_dry_run_safe) { + char source_directory[] = "/tmp/cbm_ui_lifecycle_source_XXXXXX"; + char install_directory[] = "/tmp/cbm_ui_lifecycle_target_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(source_directory)); + ASSERT_NOT_NULL(cbm_mkdtemp(install_directory)); + + size_t pack_length = 0U; + unsigned char *pack = ui_test_pack(&pack_length, false); + ASSERT_NOT_NULL(pack); + char source_pack[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(source_directory, pack, pack_length, source_pack)); + const char *pack_name = strrchr(source_pack, '/'); + ASSERT_NOT_NULL(pack_name); + pack_name++; + char installed_pack[1024]; + ASSERT_LT( + snprintf(installed_pack, sizeof(installed_pack), "%s/%s", install_directory, pack_name), + (int)sizeof(installed_pack)); + + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", source_directory, 1), 0); + char error[512] = {0}; + bool dry_install = cbm_ui_assets_install(install_directory, true, error, sizeof(error)); + cbm_path_info_t info; + bool absent_after_dry_install = cbm_path_info_utf8(installed_pack, &info) != 0; + bool installed = cbm_ui_assets_install(install_directory, false, error, sizeof(error)); + + unsigned char *installed_bytes = malloc(pack_length); + FILE *installed_file = cbm_fopen(installed_pack, "rb"); + size_t installed_length = installed_file && installed_bytes + ? fread(installed_bytes, 1U, pack_length, installed_file) + : 0U; + bool installed_closed = !installed_file || fclose(installed_file) == 0; + bool exact_copy = installed_bytes && installed_length == pack_length && + memcmp(installed_bytes, pack, pack_length) == 0; + free(installed_bytes); + + bool dry_remove = cbm_ui_assets_remove(install_directory, true, error, sizeof(error)); + bool present_after_dry_remove = cbm_path_info_utf8(installed_pack, &info) == 0; + bool removed = cbm_ui_assets_remove(install_directory, false, error, sizeof(error)); + bool absent_after_remove = cbm_path_info_utf8(installed_pack, &info) != 0; + + bool reinstalled = cbm_ui_assets_install(install_directory, false, error, sizeof(error)); + FILE *tampered_file = cbm_fopen(installed_pack, "r+b"); + bool tampered = tampered_file && fseek(tampered_file, -1L, SEEK_END) == 0; + int original_last_byte = tampered ? fgetc(tampered_file) : EOF; + tampered = tampered && original_last_byte != EOF && fseek(tampered_file, -1L, SEEK_END) == 0 && + fputc(original_last_byte ^ 1, tampered_file) != EOF; + if (tampered_file && fclose(tampered_file) != 0) { + tampered = false; + } + bool refused_tampered_remove = + !cbm_ui_assets_remove(install_directory, false, error, sizeof(error)); + bool tampered_preserved = cbm_path_info_utf8(installed_pack, &info) == 0; + bool preservation_explained = strstr(error, "not owned; preserved") != NULL; + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(installed_pack); + (void)cbm_unlink(source_pack); + (void)cbm_rmdir(install_directory); + (void)cbm_rmdir(source_directory); + free(pack); + + ASSERT_TRUE(dry_install); + ASSERT_TRUE(absent_after_dry_install); + ASSERT_TRUE(installed); + ASSERT_TRUE(installed_closed); + ASSERT_TRUE(exact_copy); + ASSERT_TRUE(dry_remove); + ASSERT_TRUE(present_after_dry_remove); + ASSERT_TRUE(removed); + ASSERT_TRUE(absent_after_remove); + ASSERT_TRUE(reinstalled); + ASSERT_TRUE(tampered); + ASSERT_TRUE(refused_tampered_remove); + ASSERT_TRUE(tampered_preserved); + ASSERT_TRUE(preservation_explained); + PASS(); +} + +TEST(ui_asset_pack_staged_removal_is_owned_and_transactional) { + char source_directory[] = "/tmp/cbm_ui_staged_remove_source_XXXXXX"; + char install_directory[] = "/tmp/cbm_ui_staged_remove_target_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(source_directory)); + ASSERT_NOT_NULL(cbm_mkdtemp(install_directory)); + size_t pack_length = 0U; + unsigned char *pack = ui_test_pack(&pack_length, false); + ASSERT_NOT_NULL(pack); + char source_pack[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(source_directory, pack, pack_length, source_pack)); + const char *pack_name = strrchr(source_pack, '/'); + ASSERT_NOT_NULL(pack_name); + pack_name++; + char installed_pack[1024]; + ASSERT_LT( + snprintf(installed_pack, sizeof(installed_pack), "%s/%s", install_directory, pack_name), + (int)sizeof(installed_pack)); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", source_directory, 1), 0); + char error[512] = {0}; + ASSERT_TRUE(cbm_ui_assets_install(install_directory, false, error, sizeof(error))); + + cbm_activation_transaction_t *removal = NULL; + bool foreign = false; + bool staged = + cbm_ui_assets_stage_remove(install_directory, &removal, &foreign, error, sizeof(error)); + ASSERT_TRUE(staged); + ASSERT_FALSE(foreign); + ASSERT_NOT_NULL(removal); + ASSERT_EQ(cbm_activation_transaction_commit(removal, NULL, NULL), + CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_EQ(cbm_activation_transaction_finalize(removal), CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_EQ(cbm_activation_transaction_close(&removal), CBM_ACTIVATION_TRANSACTION_OK); + cbm_path_info_t info; + bool exact_removed = cbm_path_info_utf8(installed_pack, &info) != 0; + + ASSERT_TRUE(cbm_ui_assets_install(install_directory, false, error, sizeof(error))); + FILE *tampered_file = cbm_fopen(installed_pack, "r+b"); + ASSERT_NOT_NULL(tampered_file); + ASSERT_EQ(fseek(tampered_file, -1L, SEEK_END), 0); + int last_byte = fgetc(tampered_file); + ASSERT_TRUE(last_byte != EOF); + ASSERT_EQ(fseek(tampered_file, -1L, SEEK_END), 0); + ASSERT_TRUE(fputc(last_byte ^ 1, tampered_file) != EOF); + ASSERT_EQ(fclose(tampered_file), 0); + removal = NULL; + foreign = false; + bool foreign_staged = + cbm_ui_assets_stage_remove(install_directory, &removal, &foreign, error, sizeof(error)); + bool foreign_preserved = cbm_path_info_utf8(installed_pack, &info) == 0; + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_activation_transaction_close(&removal); + (void)cbm_unlink(installed_pack); + (void)cbm_unlink(source_pack); + (void)cbm_rmdir(install_directory); + (void)cbm_rmdir(source_directory); + free(pack); + + ASSERT_TRUE(exact_removed); + ASSERT_TRUE(foreign_staged); + ASSERT_TRUE(foreign); + ASSERT_NULL(removal); + ASSERT_TRUE(foreign_preserved); + PASS(); +} + +/* File identity alone does not detect a truncate/rewrite through an already + * open handle. Publication must validate the exact bytes at the public name, + * and removal must validate the exact object after moving it to the private + * retained name. Both mutations below preserve the inode on POSIX. */ +TEST(ui_asset_pack_commit_rejects_in_place_rewrite_after_stage) { + char source_directory[] = "/tmp/cbm_ui_commit_source_XXXXXX"; + char install_directory[] = "/tmp/cbm_ui_commit_target_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(source_directory)); + ASSERT_NOT_NULL(cbm_mkdtemp(install_directory)); + size_t pack_length = 0U; + unsigned char *pack = ui_test_pack(&pack_length, false); + ASSERT_NOT_NULL(pack); + char source_pack[1024]; + cbm_ui_assets_reset_for_testing(); + ASSERT_TRUE(ui_test_publish_pack(source_directory, pack, pack_length, source_pack)); + const char *pack_name = strrchr(source_pack, '/'); + ASSERT_NOT_NULL(pack_name); + pack_name++; + char installed_pack[1024]; + ASSERT_LT( + snprintf(installed_pack, sizeof(installed_pack), "%s/%s", install_directory, pack_name), + (int)sizeof(installed_pack)); + char *old_assets = getenv("CBM_UI_ASSETS_DIR") ? strdup(getenv("CBM_UI_ASSETS_DIR")) : NULL; + ASSERT_EQ(cbm_setenv("CBM_UI_ASSETS_DIR", source_directory, 1), 0); + char error[512] = {0}; + ASSERT_TRUE(cbm_ui_assets_install(install_directory, false, error, sizeof(error))); + ASSERT_TRUE(cbm_ui_assets_verify_file(installed_pack)); + + cbm_activation_transaction_t *publication = NULL; + ASSERT_TRUE( + cbm_ui_assets_stage_install(install_directory, &publication, error, sizeof(error))); + ASSERT_NOT_NULL(publication); + const char *staged = cbm_activation_transaction_staged_path(publication); + ASSERT_NOT_NULL(staged); + FILE *staged_file = cbm_fopen(staged, "r+b"); + ASSERT_NOT_NULL(staged_file); + ASSERT_EQ(fseek(staged_file, -1L, SEEK_END), 0); + int staged_last = fgetc(staged_file); + ASSERT_TRUE(staged_last != EOF); + ASSERT_EQ(fseek(staged_file, -1L, SEEK_END), 0); + ASSERT_TRUE(fputc(staged_last ^ 1, staged_file) != EOF); + ASSERT_EQ(fclose(staged_file), 0); + ASSERT_EQ(cbm_ui_assets_commit_install(publication), + CBM_ACTIVATION_TRANSACTION_VALIDATION_FAILED); + ASSERT_EQ(cbm_activation_transaction_close(&publication), CBM_ACTIVATION_TRANSACTION_OK); + ASSERT_TRUE(cbm_ui_assets_verify_file(installed_pack)); + + cbm_activation_transaction_t *removal = NULL; + bool foreign = false; + ASSERT_TRUE(cbm_ui_assets_stage_remove(install_directory, &removal, &foreign, error, + sizeof(error))); + ASSERT_FALSE(foreign); + ASSERT_NOT_NULL(removal); + FILE *target_file = cbm_fopen(installed_pack, "r+b"); + ASSERT_NOT_NULL(target_file); + ASSERT_EQ(fseek(target_file, -1L, SEEK_END), 0); + int target_last = fgetc(target_file); + ASSERT_TRUE(target_last != EOF); + ASSERT_EQ(fseek(target_file, -1L, SEEK_END), 0); + ASSERT_TRUE(fputc(target_last ^ 1, target_file) != EOF); + ASSERT_EQ(fclose(target_file), 0); + ASSERT_EQ(cbm_ui_assets_commit_removal(removal), + CBM_ACTIVATION_TRANSACTION_VALIDATION_FAILED); + ASSERT_EQ(cbm_activation_transaction_close(&removal), CBM_ACTIVATION_TRANSACTION_OK); + cbm_path_info_t preserved_info; + ASSERT_EQ(cbm_path_info_utf8(installed_pack, &preserved_info), 0); + ASSERT_TRUE(preserved_info.is_regular); + ASSERT_FALSE(cbm_ui_assets_verify_file(installed_pack)); + + cbm_ui_assets_reset_for_testing(); + ui_test_restore_assets_env(old_assets); + (void)cbm_unlink(installed_pack); + (void)cbm_unlink(source_pack); + (void)cbm_rmdir(install_directory); + (void)cbm_rmdir(source_directory); + free(pack); PASS(); } -TEST(embedded_stub_count) { - /* Stub should have 0 files */ - ASSERT_EQ(CBM_EMBEDDED_FILE_COUNT, 0); +TEST(ui_asset_standard_manifest_has_no_capability) { + cbm_ui_assets_reset_for_testing(); + ASSERT_FALSE(cbm_ui_assets_supported()); + ASSERT_EQ(cbm_ui_assets_state(), CBM_UI_ASSETS_UNAVAILABLE); + ASSERT_NULL(cbm_ui_asset_lookup("/index.html")); PASS(); } @@ -870,9 +1398,16 @@ SUITE(ui) { RUN_TEST(config_corrupt_file); RUN_TEST(config_missing_fields); - /* Embedded assets (stub) */ - RUN_TEST(embedded_lookup_not_found); - RUN_TEST(embedded_stub_count); + /* External UI asset pack */ + RUN_TEST(ui_asset_pack_validates_before_publication); + RUN_TEST(ui_asset_pack_wrong_hash_fails_closed); + RUN_TEST(ui_asset_pack_structural_corruption_fails_after_valid_hash); + RUN_TEST(ui_asset_pack_prestart_cancellation_is_not_lost); + RUN_TEST(ui_asset_pack_install_never_follows_a_predictable_temp_symlink); + RUN_TEST(ui_asset_pack_install_remove_is_hash_bound_and_dry_run_safe); + RUN_TEST(ui_asset_pack_staged_removal_is_owned_and_transactional); + RUN_TEST(ui_asset_pack_commit_rejects_in_place_rewrite_after_stage); + RUN_TEST(ui_asset_standard_manifest_has_no_capability); /* Layout engine */ RUN_TEST(layout_empty_graph); diff --git a/tests/test_ui_asset_pack_contract.sh b/tests/test_ui_asset_pack_contract.sh new file mode 100644 index 0000000000..6927027012 --- /dev/null +++ b/tests/test_ui_asset_pack_contract.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Contract for the external UI asset pack. The pack deliberately stays +# uncompressed and independently scannable: this is an asset boundary, not an +# attempt to hide frontend bytes from security tooling. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/cbm-ui-pack-test.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT + +mkdir -p "$WORK/a/assets" "$WORK/b/assets" +printf '%s\n' 'Codebase Memory — Graph' > "$WORK/a/index.html" +printf '%s\n' 'console.log("transparent-ui-pack")' > "$WORK/a/assets/app.js" +printf '%s\n' 'body { color: #fff; }' > "$WORK/a/assets/app.css" + +# Copy in a deliberately different creation order. Filesystem enumeration, +# mtimes, and source-root names must not affect the output. +cp "$WORK/a/assets/app.css" "$WORK/b/assets/app.css" +cp "$WORK/a/index.html" "$WORK/b/index.html" +cp "$WORK/a/assets/app.js" "$WORK/b/assets/app.js" +touch -t 200101010101 "$WORK/a/index.html" +touch -t 203001010101 "$WORK/b/index.html" + +mkdir -p "$WORK/a-out" "$WORK/b-out" +node "$ROOT/scripts/pack-ui-assets.mjs" \ + "$WORK/a" "$WORK/a-out" "$WORK/a-manifest.c" +node "$ROOT/scripts/pack-ui-assets.mjs" \ + "$WORK/b" "$WORK/b-out" "$WORK/b-manifest.c" + +PACK_A="$(find "$WORK/a-out" -maxdepth 1 -type f -name 'cbm-ui-*.pack')" +PACK_B="$(find "$WORK/b-out" -maxdepth 1 -type f -name 'cbm-ui-*.pack')" +[ -n "$PACK_A" ] && [ -n "$PACK_B" ] +[ "$(basename "$PACK_A")" = "$(basename "$PACK_B")" ] +cmp "$PACK_A" "$PACK_B" +cmp "$WORK/a-manifest.c" "$WORK/b-manifest.c" + +python3 - "$PACK_A" "$WORK/a-manifest.c" <<'PY' +from __future__ import annotations + +import hashlib +import pathlib +import re +import struct +import sys + +pack_path = pathlib.Path(sys.argv[1]) +manifest_path = pathlib.Path(sys.argv[2]) +data = pack_path.read_bytes() +manifest = manifest_path.read_text(encoding="utf-8") + +( + magic, + version, + header_bytes, + flags, + count, + entry_bytes, + index_offset, + index_size, + paths_offset, + paths_size, + payload_offset, + payload_size, + total_size, +) = struct.unpack_from("<8sHHIIIQQQQQQQ", data, 0) +assert magic == b"CBMUIPK\0", "wrong pack magic" +assert version == 1 +assert header_bytes == 80 +assert flags == 0 +assert count == 3 +assert entry_bytes == 24 +assert index_offset == 80 +assert index_size == count * entry_bytes +assert paths_offset == index_offset + index_size +assert payload_offset == paths_offset + paths_size +assert payload_size == len(data) - payload_offset +assert total_size == len(data) + +cursor = index_offset +entries = [] +for _ in range(count): + path_offset, path_len, mime_id, cache_id, offset, length = struct.unpack_from( + "" in data + +digest = hashlib.sha256(data).hexdigest() +assert re.search(rf'CBM_UI_ASSET_SHA256\[\]\s*=\s*"{digest}"', manifest) +assert re.search(rf"CBM_UI_ASSET_SIZE\s*=\s*UINT64_C\({len(data)}\)", manifest) +assert re.search( + rf'CBM_UI_ASSET_PACK_NAME\[\]\s*=\s*"cbm-ui-{digest}\.pack"', manifest +) +assert pack_path.name == f"cbm-ui-{digest}.pack" +PY + +# A source symlink makes the build depend on bytes outside the declared dist +# tree. It is rejected rather than followed or silently omitted. +ln -s ../index.html "$WORK/a/assets/linked.html" +if node "$ROOT/scripts/pack-ui-assets.mjs" \ + "$WORK/a" "$WORK/a-out" "$WORK/rejected.c" 2>/dev/null; then + echo "FAIL: UI pack generator accepted a symlink" >&2 + exit 1 +fi + +echo "PASS: deterministic, transparent, hash-bound UI asset pack" diff --git a/tests/test_vendored_integrity_contract.sh b/tests/test_vendored_integrity_contract.sh new file mode 100644 index 0000000000..c397685271 --- /dev/null +++ b/tests/test_vendored_integrity_contract.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Contract: every vendored input is content-pinned, including opaque blobs and +# the generated Tree-sitter sources that are compiled into release binaries. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +FIXTURE="$(mktemp -d "${TMPDIR:-/tmp}/cbm-vendored-integrity.XXXXXX")" +trap 'rm -rf "$FIXTURE"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +make_fixture() { + local destination="$1" + mkdir -p "$destination/scripts" "$destination/vendored/yyjson" + cp "$ROOT/scripts/security-vendored.sh" "$destination/scripts/security-vendored.sh" + : > "$destination/vendored/yyjson/safe.c" + printf '%s %s\n' \ + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' \ + 'vendored/yyjson/safe.c' > "$destination/scripts/vendored-checksums.txt" +} + +opaque="$FIXTURE/opaque" +make_fixture "$opaque" +printf '\230\237\000\000opaque-vector-bytes\n' > "$opaque/vendored/yyjson/code_vectors.bin" +if (cd "$opaque" && bash scripts/security-vendored.sh >/dev/null 2>&1); then + fail "vendored integrity gate accepted an unmanifested opaque binary blob" +fi + +grammar="$FIXTURE/grammar" +make_fixture "$grammar" +mkdir -p "$grammar/internal/cbm/vendored/grammars/example" +printf '%s\n' 'int tree_sitter_example(void) { return 1; }' \ + > "$grammar/internal/cbm/vendored/grammars/example/parser.c" +if (cd "$grammar" && bash scripts/security-vendored.sh >/dev/null 2>&1); then + fail "vendored integrity gate accepted an unmanifested generated grammar source" +fi + +echo "vendored integrity contract: OK" diff --git a/tests/test_venue_parity_contract.sh b/tests/test_venue_parity_contract.sh index 11cac18dae..48c4ea2ef6 100755 --- a/tests/test_venue_parity_contract.sh +++ b/tests/test_venue_parity_contract.sh @@ -99,7 +99,7 @@ ALLOWED_CMDS = { "gh", "python3", "node", "codesign", "xcrun", "command", "awk", } # Per-file additions: _build.yml packages artifacts (make/strip are packaging -# steps, not test legs) and drives npm for the embedded UI; release.yml pushes finished artifacts to registries after +# steps, not test legs) and drives npm for the UI build; release.yml pushes finished artifacts to registries after # every product-exercising gate already ran; the brew tap smoke installs the # released formula. ALLOWED_EXTRA = { diff --git a/tests/test_vm_worktree_manifest.sh b/tests/test_vm_worktree_manifest.sh index d858c27377..9dcfc81c84 100644 --- a/tests/test_vm_worktree_manifest.sh +++ b/tests/test_vm_worktree_manifest.sh @@ -60,6 +60,26 @@ if ! grep -Fq -- '-c \"set -e -o pipefail; $*\"' "$driver"; then echo "FAIL: Windows VM remote shell must propagate pipeline failures" >&2 exit 1 fi +if grep -Fq 'cd $VM_REPO && git apply' <<<"$sync_block" || + grep -Fq 'cd $VM_REPO && tar -xf' <<<"$sync_block"; then + echo "FAIL: Windows VM sync must expand the selected checkout locally before transport" >&2 + exit 1 +fi +if ! grep -Fq "cd '\$VM_REPO' && git apply" <<<"$sync_block" || + ! grep -Fq "cd '\$VM_REPO' && tar -xf" <<<"$sync_block"; then + echo "FAIL: Windows VM sync patch and manifest must target the selected checkout" >&2 + exit 1 +fi +if ! grep -Fq 'CBM_CI_RUN_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$' "$driver"; then + echo "FAIL: Windows VM run ids must be validated before remote path interpolation" >&2 + exit 1 +fi +preflight_block="$(sed -n '/^vm_preflight() {$/,/^}$/p' "$driver")" +if grep -Fq 'C:\\cbm\\scripts\\ci' <<<"$preflight_block" || + [ "$(grep -c 'VM_REPO_WIN' <<<"$preflight_block")" -lt 2 ]; then + echo "FAIL: Windows VM preflight must use the selected per-run checkout" >&2 + exit 1 +fi if ! grep -Fq 'tests/test_vm_worktree_manifest.sh' "$ROOT/scripts/test.sh"; then echo "FAIL: Windows VM sync contract is not wired into the test suite" >&2 exit 1 @@ -116,7 +136,7 @@ if ! grep -Fq "JOBS='\$(nproc)'" "$provisioner" || exit 1 fi guards_block="$(sed -n '/^guards)/,/^ ;;/p' "$driver")" -# The guards leg builds the CI-shaped product binary (embedded UI) into an +# The guards leg builds the CI-shaped UI product and its external pack into an # isolated BUILD_DIR, then hands that artifact to the maintained # PowerShell driver through a plain cmd shell (CI's environment shape, not # the MSYS2 login shell whose TMP ancestry the daemon correctly refuses). @@ -128,6 +148,11 @@ if ! grep -Fq 'scripts/build.sh --with-ui CC=clang CXX=clang++ SANITIZE= BUILD_D echo "FAIL: Windows VM guards must delegate to the maintained native-Windows driver" >&2 exit 1 fi +if ! grep -Fq 'cd /d ${VM_REPO_WIN}' <<<"$guards_block" || + grep -Fq 'cd /d C:\\cbm' <<<"$guards_block"; then + echo "FAIL: Windows VM guards must run against the selected per-run checkout" >&2 + exit 1 +fi if grep -Fq 'for g in tests/windows/test_*.py' <<<"$guards_block"; then echo "FAIL: Windows VM guards must not duplicate the native-Windows driver loop" >&2 exit 1 diff --git a/tests/test_vt_gate_zero_tolerance_contract.sh b/tests/test_vt_gate_zero_tolerance_contract.sh index fb5d2177f4..f7c29a3407 100644 --- a/tests/test_vt_gate_zero_tolerance_contract.sh +++ b/tests/test_vt_gate_zero_tolerance_contract.sh @@ -1,15 +1,7 @@ #!/usr/bin/env bash -# Contract: the VirusTotal release gate (scripts/ci/check-virustotal.sh) is -# ZERO tolerance — ANY detection on ANY artifact blocks, with no version -# carve-out, no engine carve-out, and no evidence side-channel. -# -# A tolerance path existed briefly (single-engine Microsoft "!ml" on -# pre-releases, endpoint-evidence-gated) and was deliberately reverted: the -# project does not ship binaries carrying a VirusTotal detection, full stop. -# False positives are resolved upstream (Microsoft FP submission for the exact -# hashes, then re-run the verify job — no rebuild), never by loosening the -# gate. This contract pins that decision against a stubbed VT API so a future -# "just tolerate this one heuristic" change has to consciously delete it. +# Contract: the VT gate is an exact-set, content-bound, minimum-engine, +# zero-tolerance gate. Empty, partial, duplicate, unexpected or malformed +# action output fails before a release can be described as scanned. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -18,97 +10,256 @@ FIX="$(mktemp -d "${TMPDIR:-/tmp}/vt-gate-contract.XXXXXX")" trap 'rm -rf "$FIX"' EXIT fail() { echo "FAIL: $*" >&2; exit 1; } +hash_file() { + sha256sum "$1" 2>/dev/null | awk '{print $1}' || shasum -a 256 "$1" | awk '{print $1}' +} -# Tripwire for the specific reverted mechanism (behavioral checks below are -# the real contract; these names only catch that exact machinery returning). -for needle in defender-endpoint-verification av-endpoint-verify; do +for needle in defender-endpoint-verification av-endpoint-verify false-positive false_positive; do if grep -q "$needle" "$GATE"; then - fail "gate references '$needle' — the reverted evidence side-channel is back" + fail "gate references reverted/tolerance machinery '$needle'" fi done -mkdir -p "$FIX/bin" "$FIX/work/binaries" "$FIX/responses" +mkdir -p "$FIX/bin" "$FIX/work/binaries/objects" "$FIX/responses" +printf 'release-bytes\n' > "$FIX/work/binaries/objects/probe" +printf 'second-object\n' > "$FIX/work/binaries/objects/probe2" +PROBE_SHA="$(hash_file "$FIX/work/binaries/objects/probe")" +PROBE2_SHA="$(hash_file "$FIX/work/binaries/objects/probe2")" +PROBE_SIZE="$(wc -c < "$FIX/work/binaries/objects/probe" | tr -d ' ')" +PROBE2_SIZE="$(wc -c < "$FIX/work/binaries/objects/probe2" | tr -d ' ')" + +write_manifest() { # one|two + local count=1 associations=3 + if [ "$1" = "two" ]; then count=2; associations=4; fi + { + echo '# cbm-release-scan-set-v1' + echo "# scan_objects=$count" + echo "# associations=$associations" + printf 'scan_path\tsha256\tsize\tassociation_count\tassociation_kinds\n' + printf 'objects/probe\t%s\t%s\t3\tarchive,binary,runtime\n' "$PROBE_SHA" "$PROBE_SIZE" + if [ "$1" = "two" ]; then + printf 'objects/probe2\t%s\t%s\t1\tarchive\n' "$PROBE2_SHA" "$PROBE2_SIZE" + fi + } > "$FIX/work/binaries/scan-set.tsv" + { + echo '# cbm-release-scan-associations-v2' + echo "# associations=$associations" + echo "# scan_objects=$count" + printf 'association_type\tarchive\tarchive_sha256\tvariant\tkind\tmember\tasset_path\tmime\tscan_path\tobject_sha256\tsize\n' + printf 'archive\tprobe.tar.gz\t%s\tstandard\tarchive\t\t\t\tobjects/probe\t%s\t%s\n' "$PROBE_SHA" "$PROBE_SHA" "$PROBE_SIZE" + printf 'member\tprobe.tar.gz\t%s\tstandard\tbinary\tcodebase-memory-mcp\t\t\tobjects/probe\t%s\t%s\n' "$PROBE_SHA" "$PROBE_SHA" "$PROBE_SIZE" + printf 'member\tprobe.tar.gz\t%s\tstandard\truntime\tLICENSE\t\t\tobjects/probe\t%s\t%s\n' "$PROBE_SHA" "$PROBE_SHA" "$PROBE_SIZE" + if [ "$1" = "two" ]; then + printf 'archive\tprobe2.zip\t%s\tstandard\tarchive\t\t\t\tobjects/probe2\t%s\t%s\n' "$PROBE2_SHA" "$PROBE2_SHA" "$PROBE2_SIZE" + fi + } > "$FIX/work/binaries/associations.tsv" +} +write_manifest one -# curl stub: serve the canned VT analysis JSON selected by the id in the URL. +# curl stub: serve canned analysis JSON selected by the ID in the API URL. cat > "$FIX/bin/curl" <<'EOF' #!/usr/bin/env bash url="${@: -1}" id="${url##*/analyses/}" -resp="$STUB_RESPONSES/$id.json" -[ -f "$resp" ] || exit 22 -cat "$resp" +response="$STUB_RESPONSES/$id.json" +[ -f "$response" ] || exit 22 +cat "$response" EOF chmod +x "$FIX/bin/curl" -mk_response() { # id malicious suspicious +write_response() { # id malicious suspicious undetected sha size cat > "$FIX/responses/$1.json" < "$FIX/responses/named_engine.json" <<'EOF' -{"data":{"attributes":{"status":"completed", - "stats":{"malicious":1,"suspicious":0,"undetected":60,"harmless":0}, - "results":{ - "Microsoft":{"category":"malicious","result":"Trojan:Win32/Wacatac.B!ml", - "engine_version":"1.1.24010.10","engine_update":"20260807"}, - "Kaspersky":{"category":"undetected","result":null, - "engine_version":"22.0.1.28","engine_update":"20260807"} - }}}} +write_response clean-analysis 0 0 60 "$PROBE_SHA" "$PROBE_SIZE" +write_response clean-analysis-2 0 0 60 "$PROBE2_SHA" "$PROBE2_SIZE" +write_response one-malicious 1 0 59 "$PROBE_SHA" "$PROBE_SIZE" +write_response one-suspicious 0 1 59 "$PROBE_SHA" "$PROBE_SIZE" +write_response low-engines 0 0 49 "$PROBE_SHA" "$PROBE_SIZE" +write_response wrong-hash 0 0 60 "$PROBE2_SHA" "$PROBE_SIZE" +write_response wrong-size 0 0 60 "$PROBE_SHA" "$((PROBE_SIZE + 1))" + +cat > "$FIX/responses/named-engine.json" < "$FIX/responses/malformed-stats.json" < "$FIX/responses/missing-analysis-id.json" < "$FIX/responses/missing-microsoft.json" < "$FIX/responses/microsoft-timeout.json" < "$FIX/responses/wrong-response-type.json" < "$FIX/responses/missing-malicious-stat.json" < "$FIX/responses/detail-aggregate-mismatch.json" < "$FIX/responses/malformed-microsoft-evidence.json" < "$FIX/work/binaries/probe" +url_for() { printf 'https://www.virustotal.com/gui/file-analysis/%s/detection' "$1"; } -run_gate() { # version analysis-id -> echoes exit code - rm -f /tmp/vt_gate_fail +run_gate() { # action-output -> prints rc + rm -f "$FIX/work/binaries/vt-results.tsv" "$FIX/last.log" local rc=0 (cd "$FIX/work" && PATH="$FIX/bin:$PATH" \ STUB_RESPONSES="$FIX/responses" \ - VT_API_KEY="stub" \ - VERSION="$1" \ - VT_ANALYSIS="binaries/probe=https://www.virustotal.com/gui/file-analysis/$2/detection" \ - bash "$GATE" >"$FIX/last.log" 2>&1) || rc=$? + VT_API_KEY=stub \ + VT_ANALYSIS="$1" \ + VT_EXPECTED_SCAN_SET=binaries/scan-set.tsv \ + VT_ASSOCIATIONS=binaries/associations.tsv \ + VT_RESULTS_PATH=binaries/vt-results.tsv \ + VT_REQUEST_INTERVAL_SECONDS=0 \ + VT_POLL_TIMEOUT_SECONDS=2 \ + VT_CURL_TIMEOUT_SECONDS=1 \ + MIN_ENGINES=50 \ + bash "$GATE" > "$FIX/last.log" 2>&1) || rc=$? echo "$rc" } -[ "$(run_gate v0.9.1 clean)" = "0" ] || fail "clean scan must pass" +clean_output="binaries/objects/probe=$(url_for clean-analysis)" +[ "$(run_gate "$clean_output")" = "0" ] || fail "clean exact scan must pass: $(cat "$FIX/last.log")" +RESULTS="$FIX/work/binaries/vt-results.tsv" +[ -f "$RESULTS" ] || fail "clean gate did not atomically publish its results manifest" +grep -q '^# cbm-virustotal-results-v1$' "$RESULTS" || fail "results marker missing" +grep -q "objects/probe.*$PROBE_SHA.*$PROBE_SIZE.*60" "$RESULTS" || \ + fail "results do not bind path/hash/size/actual engine count" -for version in v0.9.1 v0.9.1-rc.1 v0.9.1-pre1 v1.0.0-alpha.1; do - [ "$(run_gate "$version" one_malicious)" != "0" ] || \ - fail "1 malicious must block on $version (pre-release carve-outs are gone)" - [ "$(run_gate "$version" one_suspicious)" != "0" ] || \ - fail "1 suspicious must block on $version" +for id in one-malicious one-suspicious; do + [ "$(run_gate "binaries/objects/probe=$(url_for "$id")")" != "0" ] || \ + fail "$id must block" done +[ "$(run_gate "binaries/objects/probe=$(url_for low-engines)")" != "0" ] || \ + fail "completed analysis below the 50-engine policy must block" +grep -q 'only 49/49 decisive engines' "$FIX/last.log" || fail "low-engine failure is not explicit" -# A detection must NAME the engine that fired. -# -# The gate reported counts only ("1 malicious / 59 engines"), which is the one -# thing a reader cannot act on: a false-positive submission goes to a SPECIFIC -# vendor (and the WDSI developer path is a web form, so there is no API to guess -# with), and triage starts with "known-noisy engine, or a new family?". In the -# 2026-08-07 dry run three artifacts flagged and the logs could not say by what, -# so the answer had to be fetched by hand from the web UI afterwards. -[ "$(run_gate v0.9.1 named_engine)" != "0" ] || fail "named-engine detection must still block" -grep -q "Microsoft" "$FIX/last.log" || \ - fail "gate must name the flagging engine (Microsoft) — got: $(cat "$FIX/last.log")" -grep -q "Trojan:Win32/Wacatac.B!ml" "$FIX/last.log" || \ - fail "gate must report the detection label, not just the vendor" -# Engines that did NOT flag must not be listed, or the actionable line drowns. -if grep -q "detected by: Kaspersky" "$FIX/last.log"; then - fail "gate listed a non-flagging engine (Kaspersky) as a detection" +[ "$(run_gate "binaries/objects/probe=$(url_for named-engine)")" != "0" ] || \ + fail "named-engine detection must block" +grep -q 'Microsoft = Trojan:Script/Wacatac.B!ml' "$FIX/last.log" || \ + fail "gate must report the flagging engine and label" +grep -q $'objects/probe\t.*\t1\t0\t' "$FIX/work/binaries/vt-results.tsv" || \ + fail "blocked completed-set counts were not preserved as workflow evidence" +if grep -q 'detected by: Kaspersky' "$FIX/last.log"; then + fail "gate listed a non-flagging engine as a detection" fi -# Naming engines must not weaken the gate: an analysis WITHOUT per-engine -# results still blocks on the counts alone. -[ "$(run_gate v0.9.1 one_malicious)" != "0" ] || \ - fail "detection without engine detail must still block" +for id in wrong-hash wrong-size malformed-stats missing-analysis-id missing-microsoft microsoft-timeout wrong-response-type missing-malicious-stat detail-aggregate-mismatch malformed-microsoft-evidence; do + [ "$(run_gate "binaries/objects/probe=$(url_for "$id")")" != "0" ] || \ + fail "$id response must fail closed" +done + + +# The independently validated association rows, not a mutable summary cell, +# decide whether an object is executable and therefore requires Microsoft. +write_manifest one +sed -i.bak 's/archive,binary,runtime/archive,runtime/' "$FIX/work/binaries/scan-set.tsv" +rm -f "$FIX/work/binaries/scan-set.tsv.bak" +[ "$(run_gate "$clean_output")" != "0" ] || \ + fail "scan-set kinds that contradict association rows must block" +grep -q 'association kinds differ' "$FIX/last.log" || fail "kind mismatch is not diagnosable" +write_manifest one + +# Exact TSV schemas reject surplus cells instead of allowing DictReader to +# hide them under its implicit None key. +printf '\textra-cell\n' >> "$FIX/work/binaries/scan-set.tsv" +[ "$(run_gate "$clean_output")" != "0" ] || fail "surplus scan-set cells must block" +write_manifest one + +# The action output itself is part of the gate. It must be a bijection with the +# expected set before any per-analysis verdict can matter. +write_manifest two +two_output="binaries/objects/probe=$(url_for clean-analysis),binaries/objects/probe2=$(url_for clean-analysis-2)" +[ "$(run_gate "$two_output")" = "0" ] || fail "two-object exact set must pass: $(cat "$FIX/last.log")" +[ "$(run_gate "binaries/objects/probe=$(url_for clean-analysis)")" != "0" ] || \ + fail "partial action output must block" +[ "$(run_gate "$two_output,binaries/objects/probe=$(url_for one-malicious)")" != "0" ] || \ + fail "duplicate action path must block" +[ "$(run_gate "$two_output,binaries/objects/unexpected=$(url_for one-malicious)")" != "0" ] || \ + fail "unexpected action path must block" +[ "$(run_gate "binaries/objects/probe")" != "0" ] || fail "malformed action entry must block" +[ "$(run_gate "binaries/objects/probe=$(url_for clean-analysis),")" != "0" ] || \ + fail "empty action entry must block" +[ "$(run_gate "binaries/objects/probe=$(url_for clean-analysis),binaries/objects/probe2=$(url_for clean-analysis)")" != "0" ] || \ + fail "one analysis URL reused for two objects must block" + +# Local bytes are rehashed immediately before matching action output, so a +# post-extraction mutation cannot inherit the original clean result. +write_manifest one +printf 'tampered\n' > "$FIX/work/binaries/objects/probe" +[ "$(run_gate "$clean_output")" != "0" ] || fail "tampered staged object must block before polling" +grep -q 'changed after extraction' "$FIX/last.log" || fail "tamper failure is not diagnosable" -rm -f /tmp/vt_gate_fail -echo "PASS: VT gate is zero tolerance and names the engine that fired" +echo 'PASS: VT gate is exact-set, content-bound, >=50-engine and zero tolerance' diff --git a/tests/test_vt_release_notes_contract.sh b/tests/test_vt_release_notes_contract.sh new file mode 100644 index 0000000000..3488fa9bb4 --- /dev/null +++ b/tests/test_vt_release_notes_contract.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Contract for idempotent, evidence-bound VirusTotal release-note publication. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT="$ROOT/scripts/ci/append-vt-notes.sh" +PUBLISH="$ROOT/scripts/ci/publish-vt-evidence.sh" +FIX="$(mktemp -d "${TMPDIR:-/tmp}/vt-notes-contract.XXXXXX")" +trap 'rm -rf "$FIX"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +mkdir -p "$FIX/bin" "$FIX/binaries" +SHA_A='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +SHA_B='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' +cat > "$FIX/binaries/associations.tsv" < "$FIX/binaries/vt-results.tsv" < "$FIX/current.md" <<'EOF' +Intro text. + + +## Security Verification + +stale data + + +Outro text. +EOF + +cat > "$FIX/bin/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [ "$1" = release ] && [ "$2" = view ]; then + cat "$STUB_RELEASE_BODY" + exit 0 +fi +if [ "$1" = release ] && [ "$2" = edit ]; then + while [ "$#" -gt 0 ]; do + if [ "$1" = --notes-file ]; then + cp "$2" "$STUB_CAPTURE" + exit 0 + fi + shift + done +fi +if [ "$1" = release ] && [ "$2" = upload ]; then + shift 3 # release upload + while [ "$#" -gt 0 ]; do + case "$1" in + --clobber) shift ;; + --repo) shift 2 ;; + *) cp "$1" "$STUB_UPLOAD_DIR/$(basename "$1")"; shift ;; + esac + done + exit 0 +fi +exit 2 +EOF +chmod +x "$FIX/bin/gh" + +run_notes() { # current capture + (cd "$FIX" && + PATH="$FIX/bin:$PATH" \ + GH_TOKEN=stub \ + VERSION=v1.0.0 \ + GITHUB_REPOSITORY=DeusData/codebase-memory-mcp \ + VT_ASSOCIATIONS=binaries/associations.tsv \ + VT_RESULTS_PATH=binaries/vt-results.tsv \ + STUB_RELEASE_BODY="$1" \ + STUB_CAPTURE="$2" \ + bash "$SCRIPT") +} + +run_notes "$FIX/current.md" "$FIX/first.md" +[ "$(grep -c 'cbm-security-verification:start' "$FIX/first.md")" = 1 ] || fail "start marker duplicated" +[ "$(grep -c 'cbm-security-verification:end' "$FIX/first.md")" = 1 ] || fail "end marker duplicated" +grep -q 'Intro text.' "$FIX/first.md" || fail "content before marked section was lost" +grep -q 'Outro text.' "$FIX/first.md" || fail "content after marked section was lost" +! grep -q 'stale data' "$FIX/first.md" || fail "stale marked section was appended instead of replaced" +grep -q '2 distinct byte objects' "$FIX/first.md" || fail "unique-object scope missing" +grep -q '3 exact release associations' "$FIX/first.md" || fail "association scope missing" +grep -q '59–61 decisive engine results' "$FIX/first.md" || fail "measured engine range missing" +grep -q 'Microsoft returned a decisive clean verdict for all 1 executable objects' "$FIX/first.md" || \ + fail "executable Microsoft coverage is not stated" +grep -q "$SHA_A" "$FIX/first.md" || fail "full archive SHA-256 missing" +grep -q "gui/file/$SHA_A/detection" "$FIX/first.md" || fail "archive VirusTotal link missing" +for asset in virustotal-associations.tsv virustotal-scan-set.tsv virustotal-results.tsv virustotal-evidence-checksums.txt; do + grep -q "releases/download/v1.0.0/$asset" "$FIX/first.md" || fail "public evidence link missing: $asset" +done +! grep -q '70+' "$FIX/first.md" || fail "unmeasured engine claim returned" + +run_notes "$FIX/first.md" "$FIX/second.md" +cmp -s "$FIX/first.md" "$FIX/second.md" || fail "marked release-note update is not idempotent" + +cp "$FIX/binaries/vt-results.tsv" "$FIX/binaries/vt-results.clean.tsv" +printf '\textra-cell\n' >> "$FIX/binaries/vt-results.tsv" +if run_notes "$FIX/current.md" "$FIX/surplus-capture.md"; then + fail "surplus result TSV cells must fail closed" +fi +mv "$FIX/binaries/vt-results.clean.tsv" "$FIX/binaries/vt-results.tsv" + +cat > "$FIX/reversed.md" <<'EOF' +Intro text. + + +stale reversed data + +EOF +if run_notes "$FIX/reversed.md" "$FIX/reversed-capture.md"; then + fail "reversed verification markers must fail closed" +fi + +mkdir -p "$FIX/public" +run_publish() { + (cd "$FIX" && + PATH="$FIX/bin:$PATH" \ + GH_TOKEN=stub \ + VERSION=v1.0.0 \ + GITHUB_REPOSITORY=DeusData/codebase-memory-mcp \ + VT_ASSOCIATIONS=binaries/associations.tsv \ + VT_EXPECTED_SCAN_SET=binaries/scan-set.tsv \ + VT_RESULTS_PATH=binaries/vt-results.tsv \ + STUB_UPLOAD_DIR="$FIX/public" \ + bash "$PUBLISH") +} +cat > "$FIX/binaries/scan-set.tsv" < "$FIX/binaries/scan-set.tsv" +if run_publish; then + fail "public evidence publication must reject a wrong scan-set marker" +fi +mv "$FIX/binaries/scan-set.clean.tsv" "$FIX/binaries/scan-set.tsv" + +echo 'PASS: VT release evidence is durable, scoped, measured, full-hash and idempotent' diff --git a/tests/test_windows_bundle_contract.sh b/tests/test_windows_bundle_contract.sh index 6942245d56..5acb8b9d51 100644 --- a/tests/test_windows_bundle_contract.sh +++ b/tests/test_windows_bundle_contract.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# INVERTED release-surface contract: Windows ships ONE binary. +# INVERTED release-surface contract: Windows ships ONE executable plus its +# authenticated runtime assets. # # History this guards against. The Windows release used to be a PAIR — a small # permanent launcher (codebase-memory-mcp.exe) plus the real product binary @@ -7,18 +8,15 @@ # reason: a running .exe cannot replace its own image on Windows, so an # in-process self-update needs a second resident binary to do the swap. # -# That stub is statically indistinguishable from a dropper — a small unsigned -# PE whose whole job is verify-and-execute another binary — and Defender's ML -# scored it Trojan:Win32/Wacatac.B!ml on x64 regardless of what we changed -# (bcrypt-free, stripped, VERSIONINFO'd and even resource-free builds were all -# flagged), while the product binary itself scans clean on every platform. +# That stub was a small unsigned PE whose whole job was verify-and-execute +# another binary: unnecessary loader-like behavior and a second artifact to +# audit. Historical variants received Microsoft Wacatac verdicts, but those +# observations did not expose a stable feature or prove classifier causation. # -# The fix removed the stub and moved self-update OUT of the process into -# install.ps1, which runs while CBM is NOT running. So this file asserts the -# ABSENCE of the flagged design, plus the one step that replaces it: -# install.ps1 must RETIRE the running binary (rename it out of the way) before -# publishing the new one — losing that step silently breaks every Windows -# update. +# The fix removed the stub. The downloaded candidate now owns the complete +# runtime-set transaction under the native activation guard; install.ps1 only +# verifies/extracts and invokes it. In particular, the script must never retire +# the old executable or publish sidecars ahead of the guarded transaction. # # The non-launcher release/security assertions from the previous contract # (VM-driver hardening, archive allowlists, HTTPS-only downloads, profile-rooted @@ -82,21 +80,25 @@ payload = "codebase-memory-mcp.payload.exe" windows_archive_names = ( binary, "cbm-integrations.json", "LICENSE", "install.ps1", "THIRD_PARTY_NOTICES.md", ) +ui_pack_pattern = r"cbm-ui-[0-9a-f]{64}\.pack" -# ── 1. The archive is exactly five files, defined in ONE place ─────────────── +# ── 1. Standard is five files; UI adds exactly one root pack ──────────────── # Every venue (release build, local artifact-flow smoke) produces archives # through scripts/package-release.sh, so the layout is asserted where it is # defined and cannot fork per venue. package_release = read("scripts/package-release.sh") -zip_call = re.search( - r"zip -q \"\$OUT_DIR/\$NAME\.zip\" \\\n(?P(?:.|\n)*?)\n", package_release +member_blocks = re.findall( + r"ARCHIVE_MEMBERS=\(\s*(?P.*?)\s*\)", package_release, re.DOTALL ) -require(zip_call is not None, "package-release.sh must build the Windows zip in one zip call") -zip_members = zip_call.group("members").split() if zip_call else [] +normalized_member_blocks = [block.split() for block in member_blocks] require( - zip_members == list(windows_archive_names), - "package-release.sh must archive EXACTLY " - f"{' '.join(windows_archive_names)} (found: {' '.join(zip_members) or 'nothing'})", + list(windows_archive_names) in normalized_member_blocks, + "package-release.sh must define the exact five-member Windows standard archive", +) +require( + package_release.count('ARCHIVE_MEMBERS+=("$UI_PACK_NAME")') == 2 + and "^cbm-ui-[0-9a-f]{64}\\.pack$" in package_release, + "package-release.sh must append exactly one hash-shaped root pack only to UI archives", ) # ── 2. No shipped surface may name the payload or build a launcher ─────────── @@ -122,8 +124,8 @@ for relative in shipped_surfaces: for marker in launcher_markers: require( marker not in source, - f"{relative} must not reference '{marker}': Windows ships ONE binary and the " - "launcher stub is what Defender flagged as Trojan:Win32/Wacatac.B!ml", + f"{relative} must not reference '{marker}': Windows ships one executable and the " + "retired launcher/payload split must not return", ) # The stub sources themselves must stay deleted, and the build must not carry a @@ -146,7 +148,7 @@ require( ) # Each archive variant is still produced through the ONE canonical packaging -# entry, so the four-file layout above governs all of them. +# entry, so the five-or-six-file layout above governs all of them. build_workflow = read(".github/workflows/_build.yml") for archive, call, ui in ( ("codebase-memory-mcp-windows-amd64.zip", "scripts/package-release.sh windows amd64", False), @@ -163,12 +165,7 @@ for archive, call, ui in ( f"_build.yml must produce {archive} via the canonical packaging entry ('{call}')", ) -# ── 3. install.ps1 retires the running binary before publishing ────────────── -# This is THE step that replaces the launcher. Windows keeps an image lock on a -# running .exe: it cannot be overwritten, but it CAN be renamed out of the way. -# install.ps1 runs while CBM is not running, renames the installed binary to -# ".retired-", then installs over the freed name. Without the -# retire step every Windows update fails on a locked destination. +# ── 3. install.ps1 delegates every runtime mutation to the candidate ───────── installer = read("install.ps1") # Windows PowerShell 5.1 decodes a BOM-less .ps1 as ANSI, so a UTF-8 em-dash # arrives as three cp1252 characters ending in a double quote. Inside a string @@ -183,43 +180,76 @@ require( + ", ".join(f"U+{ord(ch):04X}" for ch in non_ascii) + ")", ) +publish_index = installer.find("& $DownloadedBinary @InstallArgs") require( - "$Dest = Join-Path $InstallDir $BinName" in installer, - "install.ps1 must resolve the canonical install destination as $Dest", + publish_index >= 0, + "install.ps1 must invoke the downloaded candidate's native install command", ) -retire_match = re.search( - r"if \(Test-Path -LiteralPath \$Dest -PathType Leaf\) \{(?P(?:.|\n)*?)\n\}\n", - installer, +require( + "& $DownloadedBinary --version" in installer + and "& $DownloadedBinary @InstallArgs" in installer, + "install.ps1 must verify and install through the downloaded binary", ) -retire = retire_match.group("body") if retire_match else "" require( - retire_match is not None - and re.search(r"\$retired\s*=\s*\"\$Dest\.retired-", retire) is not None - and "Move-Item -LiteralPath $Dest -Destination $retired" in retire, - "install.ps1 must RETIRE the running binary (Move-Item $Dest -> $retired) before " - "publishing the new one — a running .exe cannot be overwritten in place", + "Move-Item -LiteralPath $Dest" not in installer + and ".retired-" not in installer + and "& $Dest daemon stop" not in installer, + "install.ps1 must not retire, delete, or stop the installed executable before candidate install", ) require( - retire_match is not None and "$renamed" in retire and "exit 1" in retire, - "install.ps1 must fail loudly when the running binary cannot be retired", + "Publish-CbmSidecarAtomically" not in installer + and "$AssetDest" not in installer + and "$packDestination" not in installer + and 'Get-ChildItem -LiteralPath $InstallDir -Filter "cbm-ui-*.pack"' not in installer, + "install.ps1 must not publish or garbage-collect runtime sidecars outside the native guard", ) -publish_index = installer.find("& $DownloadedBinary @InstallArgs") + +# The download workspace is recursively deleted on every exit path, so the +# installer must create it itself and must never adopt an existing name. Bind +# the reservation to this directory specifically; the updater-file publisher +# below has a separate exclusive-sibling helper. +temp_dir_factory = re.search( + r"function\s+New-CbmExclusiveTempDirectory\s*\{(?P.*?)\n\}\s*\n\s*# Detect variant", + installer, + re.DOTALL, +) +temp_dir_body = temp_dir_factory.group("body") if temp_dir_factory else "" require( - publish_index > (retire_match.start() if retire_match else -1) >= 0, - "install.ps1 must retire BEFORE it publishes the downloaded binary", + temp_dir_factory is not None + and '[guid]::NewGuid().ToString("N")' in temp_dir_body + and "for ($attempt = 0; $attempt -lt 32; $attempt++)" in temp_dir_body + and "New-Item -ItemType Directory -Path $candidate -ErrorAction Stop" in temp_dir_body + and "catch [System.IO.IOException]" in temp_dir_body + and re.search(r"New-Item[^\n]*-Force", temp_dir_body) is None + and "Remove-Item" not in temp_dir_body + and "Test-Path" not in temp_dir_body, + "install.ps1 must reserve a high-entropy temporary directory exclusively with bounded " + "collision retries, never adopt or pre-clean an existing path", ) require( - "& $DownloadedBinary --version" in installer - and "& $DownloadedBinary @InstallArgs" in installer, - "install.ps1 must verify and install through the downloaded binary", + "$TmpDir = New-CbmExclusiveTempDirectory -ParentDirectory " + "([System.IO.Path]::GetTempPath())" in installer + and '"cbm-install-$(Get-Random)"' not in installer + and "New-Item -ItemType Directory -Path $TmpDir -Force" not in installer, + "install.ps1 must clean up only the temporary directory it successfully reserved", +) + +# Bind validation to the opened handle, including its exact 64-bit size. A +# raced replacement with a valid prefix plus trailing data must be rejected. +asset_pack = read("src/ui/asset_pack.c") +require( + "handle_info.nFileSizeHigh" in asset_pack + and "handle_info.nFileSizeLow" in asset_pack + and re.search(r"handle_size\s*==\s*expected", asset_pack) is not None, + "Windows UI pack validation must compare the opened handle's 64-bit size to the manifest", ) # ── 4. Package-manager shims resolve the single Windows binary ─────────────── single_binary_contracts = { "pkg/npm/install.js": ( r"const\s+WINDOWS_BINARY_NAME\s*=\s*['\"]codebase-memory-mcp\.exe['\"]", - r"installWindowsBinaryAtomically\(", - r"windowsBinaryReady\(", + r"publishRuntimeSetWithRecovery\(", + r"runtimeSetReady\(", ), "pkg/npm/bin.js": ( r"binName\s*=\s*isWindows\s*\?\s*['\"]codebase-memory-mcp\.exe['\"]", @@ -227,7 +257,8 @@ single_binary_contracts = { ), "pkg/pypi/src/codebase_memory_mcp/_cli.py": ( r"_WINDOWS_BINARY_NAME\s*=\s*['\"]codebase-memory-mcp\.exe['\"]", - r"def\s+_windows_binary_ready\(", + r"def\s+_runtime_set_ready\(", + r"def\s+_publish_runtime_set\(", ), } for relative, patterns in single_binary_contracts.items(): @@ -241,18 +272,21 @@ for relative, patterns in single_binary_contracts.items(): f"{relative} must remain portable and not own managed launcher state", ) -# All package downloaders parse the Windows archive against the exact official -# four-root-file allowlist; they may not silently ignore an attacker-controlled -# fifth member. +# All package downloaders parse the Windows archive against an exact official +# root allowlist; direct install.ps1 additionally distinguishes standard from +# UI by the single hash-shaped pack. exact_archive_guards = { "install.ps1": ( - "$seen.Count -ne $WindowsArchiveNames.Count", + "$seen.Count -ne $expectedArchiveCount", + "$uiPackCount -ne $expectedUiPackCount", + "$UiPackPattern", '"LICENSE"', '"install.ps1"', "THIRD_PARTY_NOTICES.md", ), "pkg/npm/install.js": ( - "$seen.Count -ne $requiredNames.Count", + "seen.size !== expectedCount", + "UI_PACK_PATTERN", "WINDOWS_BINARY_NAME", "'LICENSE'", "'install.ps1'", @@ -260,7 +294,8 @@ exact_archive_guards = { ), "pkg/pypi/src/codebase_memory_mcp/_cli.py": ( "name not in required_set", - "len(seen) != len(required)", + "len(seen) != expected_count", + "_UI_PACK_RE", "_WINDOWS_BINARY_NAME", '"LICENSE"', '"install.ps1"', @@ -271,7 +306,7 @@ for relative, needles in exact_archive_guards.items(): source = read(relative) require( all(needle in source for needle in needles), - f"{relative} must reject every Windows zip namespace except the official four-file " + f"{relative} must reject every Windows zip namespace except its official exact " "allowlist", ) @@ -556,7 +591,7 @@ require( "cmd.exe", ) -# ── 8. Release smoke stays profile-rooted and single-binary ────────────────── +# ── 8. Release smoke stays profile-rooted and one-executable ───────────────── smoke_workflow = read(".github/workflows/_smoke.yml") windows_match = re.search( r"(?ms)^ smoke-windows:\s*(.*?)(?=^ [A-Za-z0-9_-]+:\s*$|\Z)", smoke_workflow @@ -607,7 +642,7 @@ require( '"$LAUNCH_DIR/codebase-memory-mcp.exe" --version', ) ), - "Windows release version checks must execute the single binary beneath the current " + "Windows release version checks must execute the runtime set's one executable beneath the current " "account profile", ) # RUNNER_TEMP is legitimate ONLY for artifact provisioning/scanning; every @@ -644,7 +679,7 @@ require( 'scripts/security-install.sh "$SECURITY_DIR/codebase-memory-mcp.exe"', ) ), - "Windows release install audit must execute the single binary beneath the current " + "Windows release install audit must execute the runtime set's one executable beneath the current " "account profile", ) @@ -652,8 +687,11 @@ require( smoke_script = read("scripts/smoke-test.sh") require( 'copy_smoke_binary "$FAKE_HOME/.local/bin/codebase-memory-mcp.exe"' not in smoke_script - and 'copy_smoke_binary "$UPDATE_HOME/.local/bin/codebase-memory-mcp.exe"' not in smoke_script, - "Windows smoke must leave canonical targets absent for an authenticated install", + and 'copy_smoke_binary "$UPDATE_HOME/.local/bin/codebase-memory-mcp.exe"' in smoke_script + and 'cp "$SMOKE_UI_PACK" "$(dirname "$destination")/$(basename "$SMOKE_UI_PACK")"' + in smoke_script, + "Windows smoke must leave the authenticated-install target absent while staging the complete " + "binary-plus-pack runtime set for the explicit update fixture", ) require( "smoke_mktemp_file" in smoke_script @@ -682,7 +720,7 @@ require( 'HOME="$WIN_HOME" TEMP="$WIN_HOME" TMP="$WIN_HOME"' in smoke_script and "MSYS2_ARG_CONV_EXCL='*'" in smoke_script and "powershell.exe -NoProfile -ExecutionPolicy Bypass -File" in smoke_script - and '"$WIN_SCRIPT" "--dir=$WIN_DIR"' in smoke_script + and '"$WIN_SCRIPT" "$DL_VARIANT_ARG" "--dir=$WIN_DIR"' in smoke_script and "& $args[1]" not in smoke_script, "Windows install.ps1 smoke must pass native HOME/TEMP/TMP and execute the script directly", ) @@ -810,10 +848,10 @@ if pr_windows_blocks: ) if failures: - print("Windows single-binary bundle contract FAILED:", file=sys.stderr) + print("Windows one-executable runtime-set contract FAILED:", file=sys.stderr) for failure in failures: print(f" - {failure}", file=sys.stderr) raise SystemExit(1) -print("Windows single-binary bundle contract passed") +print("Windows one-executable runtime-set contract passed") PY diff --git a/tests/windows/test_non_ascii_path.py b/tests/windows/test_non_ascii_path.py index 4635a36655..e82ea9d643 100644 --- a/tests/windows/test_non_ascii_path.py +++ b/tests/windows/test_non_ascii_path.py @@ -1,4 +1,4 @@ -"""GREEN regression guard — non-ASCII repo paths keep all definitions on Windows. +"""GREEN regression guard — non-ASCII runtime and repo paths work on Windows. Guards the fix for issue #636 / #357 (landed on main via #700) at the product surface (real codebase-memory-mcp process, real SQLite DB, real stdio). Two @@ -17,8 +17,8 @@ nothing. #700 routed the per-pass reads through cbm_fopen (→ _wfopen with a wide path, src/foundation/compat_fs.c), so non-ASCII paths now parse identically. -This guard fails (red) if that fix regresses. It also passes on Linux/macOS -(byte-transparent UTF-8 filesystem). +This Windows guard also exercises native runtime publication beyond legacy +MAX_PATH; it fails (red) if either path contract regresses. Exit code: 0 == invariant holds (green), 1 == invariant violated (regression), 2 == environment/setup error. @@ -29,6 +29,7 @@ import json import os import shutil +import subprocess import sys import tempfile @@ -62,6 +63,204 @@ } +def windows_extended_path(path): + """Return a Python/Win32 path that is independent of LongPathsEnabled.""" + absolute = os.path.abspath(path).replace("/", "\\") + if absolute.startswith("\\\\?\\"): + return absolute + if absolute.startswith("\\\\"): + return "\\\\?\\UNC\\" + absolute[2:] + return "\\\\?\\" + absolute + + +def run_product(argv, cwd, env, timeout=120): + options = { + "cwd": cwd, + "env": env, + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "text": True, + "timeout": timeout, + } + # With lpApplicationName=NULL, CreateProcess applies the legacy MAX_PATH + # limit while deriving the module name from the command line. Python's + # explicit `executable` argument fills lpApplicationName, allowing the + # extended path to reach the product exactly as a native launcher would. + if os.name == "nt" and argv and argv[0].startswith("\\\\?\\"): + options["executable"] = argv[0] + return subprocess.run(argv, **options) + + +def verify_relocated_runtime(binary, work): + """Probe relocation, then install/probe/uninstall beyond legacy MAX_PATH. + + This binds Windows self-discovery to GetModuleFileNameW: both the hidden + probe and a process launched from an extended-length path must find the + adjacent integration/UI assets without an ANSI-code-page fallback or an + environment override. The real install sequence also binds target + existence checks: after a secure first install, the target is rewritten in + place and a non-force `--no` install must preserve those exact bytes. + """ + source_dir = os.path.dirname(binary) + runtime_dir = os.path.join(work, "café_日本語_runtime") + os.makedirs(runtime_dir, exist_ok=True) + relocated = os.path.join(runtime_dir, os.path.basename(binary)) + shutil.copy2(binary, relocated) + + integration = os.path.join(source_dir, "cbm-integrations.json") + if not os.path.isfile(integration): + return "setup: source runtime is missing cbm-integrations.json" + shutil.copy2(integration, os.path.join(runtime_dir, "cbm-integrations.json")) + packs = [] + for name in os.listdir(source_dir): + if name.startswith("cbm-ui-") and name.endswith(".pack"): + source = os.path.join(source_dir, name) + if os.path.isfile(source): + shutil.copy2(source, os.path.join(runtime_dir, name)) + packs.append(name) + + env = os.environ.copy() + env.pop("CBM_ASSETS_DIR", None) + env.pop("CBM_UI_ASSETS_DIR", None) + env["HOME"] = os.path.join(work, "runtime_home") + env["USERPROFILE"] = env["HOME"] + env["APPDATA"] = os.path.join(work, "runtime_appdata") + env["LOCALAPPDATA"] = os.path.join(work, "runtime_localappdata") + env["XDG_CONFIG_HOME"] = os.path.join(work, "runtime_xdg_config") + env["XDG_CACHE_HOME"] = os.path.join(work, "runtime_xdg_cache") + env["XDG_DATA_HOME"] = os.path.join(work, "runtime_xdg_data") + env["CBM_CACHE_DIR"] = os.path.join(work, "runtime_cache") + for inherited_config in ( + "CLAUDE_CONFIG_DIR", "CODEX_HOME", "COPILOT_HOME", + "CRUSH_GLOBAL_CONFIG", "OPENCLAW_CONFIG_PATH", "OPENCLAW_HOME", + "OPENCLAW_PROFILE", "OPENCLAW_STATE_DIR", "OPENCLAW_WORKSPACE_DIR", + "OPENCODE_CONFIG", "OPENCODE_CONFIG_DIR", "VIBE_HOME"): + env.pop(inherited_config, None) + for directory in ( + env["HOME"], env["APPDATA"], env["LOCALAPPDATA"], + env["XDG_CONFIG_HOME"], env["XDG_CACHE_HOME"], + env["XDG_DATA_HOME"], env["CBM_CACHE_DIR"]): + os.makedirs(directory, exist_ok=True) + + def stop_runtime_daemon(): + try: + first = run_product( + [relocated, "daemon", "stop"], runtime_dir, env, timeout=30) + second = run_product( + [relocated, "daemon", "stop"], runtime_dir, env, timeout=30) + return first.returncode == 0 and second.returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + def fail(message): + stop_runtime_daemon() + return message + + probe = run_product([relocated, "--verify-runtime-assets"], runtime_dir, env) + if probe.returncode != 0: + return fail("runtime probe rc=%d output=%r packs=%r" % ( + probe.returncode, probe.stdout[-800:], packs)) + + # Keep every component below the per-component limit while making the + # complete UTF-16 path decisively longer than legacy MAX_PATH. The product, + # not Python, creates this tree during the first real install. + target = os.path.join( + work, + "install_Ωμέγα", + *("segment_%02d_%s" % (i, "x" * 32) for i in range(7)), + "bin", + ) + installed = os.path.join(target, "codebase-memory-mcp.exe") + if len(os.path.abspath(installed)) <= 260: + return fail("setup: intended long install path is only %d characters" % len( + os.path.abspath(installed))) + + first_install = run_product( + [relocated, "install", "--force", "--skip-config", "--yes", + "--dir=" + target], + runtime_dir, + env, + ) + if first_install.returncode != 0: + return fail("long-path install rc=%d output=%r path_len=%d" % ( + first_install.returncode, + first_install.stdout[-1200:], + len(os.path.abspath(installed)), + )) + + installed_extended = windows_extended_path(installed) + if not os.path.isfile(installed_extended): + return fail("long-path install did not publish %r" % installed) + integration_installed = os.path.join(target, "cbm-integrations.json") + if not os.path.isfile(windows_extended_path(integration_installed)): + return fail("long-path install omitted cbm-integrations.json") + for name in packs: + if not os.path.isfile(windows_extended_path(os.path.join(target, name))): + return fail("long-path install omitted UI pack %s" % name) + + installed_probe = run_product( + [installed_extended, "--verify-runtime-assets"], runtime_dir, env) + if installed_probe.returncode != 0: + return fail("installed long-path probe rc=%d output=%r" % ( + installed_probe.returncode, installed_probe.stdout[-1200:])) + + # Preserve the file identity/owner/DACL created by the native transaction, + # but make the bytes visibly foreign. A UTF-8-aware existence check sees + # this target and honors --no; the legacy narrow stat() treated it as absent + # and silently replaced it. + sentinel = b"cbm-long-path-existing-target-must-be-preserved\n" + with open(installed_extended, "wb") as stream: + stream.write(sentinel) + keep = run_product( + [relocated, "install", "--skip-config", "--no", "--dir=" + target], + runtime_dir, + env, + ) + if keep.returncode != 0: + return fail("long-path non-force install rc=%d output=%r" % ( + keep.returncode, keep.stdout[-1200:])) + with open(installed_extended, "rb") as stream: + retained = stream.read() + if retained != sentinel: + return fail("long-path existing target was overwritten despite --no " + "(target existence probe is not extended-path safe)") + + restore = run_product( + [relocated, "install", "--force", "--skip-config", "--yes", + "--dir=" + target], + runtime_dir, + env, + ) + if restore.returncode != 0: + return fail("long-path restore install rc=%d output=%r" % ( + restore.returncode, restore.stdout[-1200:])) + restored_probe = run_product( + [installed_extended, "--verify-runtime-assets"], runtime_dir, env) + if restored_probe.returncode != 0: + return fail("restored long-path probe rc=%d output=%r" % ( + restored_probe.returncode, restored_probe.stdout[-1200:])) + if not stop_runtime_daemon(): + return fail("could not retire the isolated daemon before uninstall") + + uninstall = run_product( + [installed_extended, "uninstall", "--yes", "--dir=" + target], + runtime_dir, + env, + ) + if uninstall.returncode != 0: + return fail("long-path uninstall rc=%d output=%r" % ( + uninstall.returncode, uninstall.stdout[-1200:])) + if os.path.exists(installed_extended): + return fail("long-path uninstall retained the installed executable") + if os.path.exists(windows_extended_path(integration_installed)): + return fail("long-path uninstall retained owned integration assets") + for name in packs: + if os.path.exists(windows_extended_path(os.path.join(target, name))): + return fail("long-path uninstall retained owned UI pack %s" % name) + stop_runtime_daemon() + return None + + def make_fixture(root): src = os.path.join(root, "src") os.makedirs(src, exist_ok=True) @@ -119,7 +318,16 @@ def main(): work = tempfile.mkdtemp(prefix="cbm_win_nonascii_") failures = [] + runtime_error = None try: + runtime_error = verify_relocated_runtime(binary, work) + if runtime_error: + print("[FAIL] non-ASCII runtime relocation: %s" % runtime_error) + print("\nREGRESSION (red): Unicode/extended-length runtime publication " + "or exact adjacent-asset resolution failed.") + return 1 + print("[PASS] Unicode relocation + extended-length install/probe/uninstall") + ascii_repo = os.path.join(work, "ascii_repo") make_fixture(ascii_repo) base = index_and_count(binary, ascii_repo, os.path.join(work, "c_ascii")) @@ -152,14 +360,15 @@ def main(): shutil.rmtree(work, ignore_errors=True) if failures: - print("\nREGRESSION (red): %d/%d non-ASCII path variants lost " + print("\nREGRESSION (red): %d/%d non-ASCII repo path variants lost " "definitions: %s" % (len(failures), len(NON_ASCII_SEGMENTS), ", ".join(failures))) print("Invariant violated: byte-identical fixtures under non-ASCII paths " "must extract the same definitions as the ASCII baseline (fixed by " "#700 — has the cbm_fopen routing in the pass readers regressed?).") return 1 - print("\nGREEN: all non-ASCII path variants matched the ASCII baseline.") + print("\nGREEN: Unicode/extended-length runtime operations passed and all " + "non-ASCII repo variants matched the ASCII baseline.") return 0 diff --git a/tests/windows/test_windows_update_handoff.py b/tests/windows/test_windows_update_handoff.py index 48155add3b..b0b36a6f44 100644 --- a/tests/windows/test_windows_update_handoff.py +++ b/tests/windows/test_windows_update_handoff.py @@ -1,14 +1,15 @@ """GREEN native-Windows guard for the `update` -> install.ps1 handoff. -Windows ships ONE binary, exactly like Linux and macOS: ``codebase-memory-mcp.exe``. +Windows ships one executable in its runtime set, exactly like Linux and macOS: +``codebase-memory-mcp.exe``. There used to be a second, permanently resident launcher stub whose only job was to swap the product binary out from under itself, because a running .exe -cannot replace its own image on Windows. Defender's ML scored that stub -Trojan:Win32/Wacatac.B!ml on x64 no matter what the build changed (bcrypt-free, -stripped, versioned and resource-free variants were all flagged), while the -product binary itself scans clean everywhere. So the swap moved OUT of the -process into install.ps1, which runs while CBM is NOT running. +cannot replace its own image on Windows. Historical variants of that stub +received Microsoft Wacatac verdicts. Those observations did not expose a +stable feature or prove causation, but the stub remained unnecessary loader- +like behavior and a second artifact to audit. The swap therefore moved OUT of +the process into install.ps1, which runs while CBM is NOT running. This guard asserts the replacement contract on real native Windows: @@ -20,8 +21,8 @@ hole. * ``update`` does not disturb an already-open MCP/daemon session. -A regression here means the in-process self-update — and therefore the -AV-flagged launcher stub — came back. +A regression here means the in-process self-update and removed launcher stub +came back. Exit code: 0 == contract honored, 1 == regression, 2 == precondition failure. From b32994868bdb4cd7f8f8b16ead1931918961bd8c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 18:13:36 +0200 Subject: [PATCH 02/13] fix(ci): audit authenticated loopback readiness Signed-off-by: Martin Vogel --- scripts/security-allowlist.txt | 13 +- scripts/security-audit.sh | 39 +- scripts/security-network-source-audit.py | 443 +++++++++++++++++++++++ 3 files changed, 458 insertions(+), 37 deletions(-) create mode 100644 scripts/security-network-source-audit.py diff --git a/scripts/security-allowlist.txt b/scripts/security-allowlist.txt index efe9b37f83..cd855e640d 100644 --- a/scripts/security-allowlist.txt +++ b/scripts/security-allowlist.txt @@ -16,13 +16,18 @@ src/daemon/bootstrap.c:fork:validated shell-free double-fork; grandchild resets src/main.c:fork:test-only parent-death containment descendant; inert unless the private CBM_TEST_WORKER_DESCENDANT_PID_FILE probe is explicitly set # ── Coordination daemon: count-bounded local IPC (never Internet sockets) ── -# Format: NETWORK:file:function:expected-count:justification -NETWORK:src/daemon/ipc.c:socket:1:single owner-only AF_UNIX stream constructor -NETWORK:src/daemon/ipc.c:connect:1:single sockaddr_un-typed connect wrapper used only with the stable owner-only daemon endpoint +# Format: NETWORK:file:function:expected-count:transport:scope:justification +# transport and scope are semantic policy, not documentation: the source audit +# rejects entries whose exact enclosing helper and call shape do not prove the +# declared local-only transport. +NETWORK:src/daemon/ipc.c:socket:1:unix:local_socket_new:single owner-only AF_UNIX stream constructor +NETWORK:src/daemon/ipc.c:connect:1:unix:local_socket_connect:single sockaddr_un-typed connect wrapper used only with the stable owner-only daemon endpoint +NETWORK:src/main.c:socket:1:loopback-v4:main_daemon_ctl_ui_endpoint_ready:single AF_INET stream client for authenticated daemon UI readiness +NETWORK:src/main.c:connect:1:loopback-v4:main_daemon_ctl_socket_connected:single sockaddr_in connect whose address is hardcoded to 127.0.0.1 and whose response must prove the daemon generation HMAC # ── Coordination daemon: count-bounded private internal file writes ──────── # Format: WRITE:file:expected-count:justification -WRITE:src/main.c:1:supervised worker writes its MCP result only to the private unique response path created and retained by its parent +WRITE:src/main.c:3:supervised worker response path; cache-scoped hook notice marker; test-seam-only browser-open marker at the caller-supplied test path WRITE:src/daemon/application.c:1:recovery truncates only its private uniquely-created per-job marker file # ── CLI: update command (user-initiated, interactive) ────────────────────── diff --git a/scripts/security-audit.sh b/scripts/security-audit.sh index ea211a40d7..a1906d915f 100755 --- a/scripts/security-audit.sh +++ b/scripts/security-audit.sh @@ -63,44 +63,17 @@ done < <(find "$ROOT/src" -name '*.c' -type f | sort) # The graph-UI HTTP server (src/ui/httpd.c) is the one component that owns a # listening socket. It is first-party, binds 127.0.0.1 only, and is audited # separately by scripts/security-ui.sh (binding + CORS checks), so it is -# exempt from this scan. The coordination transport may use only explicitly -# count-bounded AF_UNIX calls recorded in the security allow-list. +# exempt from this scan. Coordination and authenticated UI readiness may use +# only explicitly count-bounded, helper-scoped AF_UNIX or loopback-v4 calls. echo "" echo "--- Scanning for raw network calls ---" NETWORK_OK=true -while IFS= read -r file; do - relfile="${file#"$ROOT/"}" - [[ "$relfile" == "src/ui/httpd.c" ]] && continue - for func in socket connect sendto; do - matches=$(grep -n -E "[^a-z_]${func}\\(" "$file" 2>/dev/null \ - | grep -v '^\s*//' | grep -v '^\s*\*' || true) - [[ -z "$matches" ]] && continue - actual_count=$(printf '%s\n' "$matches" | wc -l | tr -d ' ') - allowance=$(grep -E "^NETWORK:${relfile}:${func}:[0-9]+:" \ - "$ALLOWLIST" 2>/dev/null || true) - allowance_count=$(printf '%s\n' "$allowance" | grep -c . || true) - expected_count=$(printf '%s\n' "$allowance" | cut -d: -f4) - semantic_ok=true - if [[ "$func" == "socket" ]] && - printf '%s\n' "$matches" | grep -v 'socket(AF_UNIX' >/dev/null; then - semantic_ok=false - fi - if [[ "$allowance_count" != "1" || - "$actual_count" != "$expected_count" || - "$semantic_ok" != "true" ]]; then - echo "BLOCKED: ${relfile}: unexpected raw ${func}() surface" - echo " expected=${expected_count:-none} actual=${actual_count} semantic_ok=${semantic_ok}" - printf '%s\n' "$matches" | sed 's/^/ /' - fail - NETWORK_OK=false - else - echo "REVIEWED: ${relfile}: ${actual_count} count-bounded ${func}() call(s)" - fi - done -done < <(find "$ROOT/src" -name '*.c' -type f | sort) - +if ! python3 "$ROOT/scripts/security-network-source-audit.py" "$ROOT" "$ALLOWLIST"; then + fail + NETWORK_OK=false +fi if $NETWORK_OK; then echo "OK: Raw network calls are absent or explicitly count-bounded." fi diff --git a/scripts/security-network-source-audit.py b/scripts/security-network-source-audit.py new file mode 100644 index 0000000000..5955cc622a --- /dev/null +++ b/scripts/security-network-source-audit.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Fail-closed source audit for raw C networking calls. + +This is intentionally a small lexical scanner rather than a grep expression: +comments and string/character literals cannot satisfy an allowance, calls at +column one or with whitespace before ``(`` are still counted, and multiple +calls on one line remain distinct. It is a review tripwire, not a substitute +for the runtime network and CodeQL gates. +""" + +from __future__ import annotations + +import dataclasses +import pathlib +import re +import sys +from typing import Sequence + + +RAW_FUNCTIONS = ("socket", "connect", "sendto") +CALL_RE = re.compile( + r"(?" + "|".join(RAW_FUNCTIONS) + r")[\t\r\n ]*\(" +) + + +@dataclasses.dataclass(frozen=True) +class Call: + name: str + line: int + start: int + end: int + text: str + + +@dataclasses.dataclass(frozen=True) +class FunctionRange: + name: str + body_start: int + body_end: int + + +@dataclasses.dataclass(frozen=True) +class Allowance: + file: str + function: str + expected: int + transport: str + scope: str + justification: str + + +def _mask_non_code(source: str) -> str: + """Replace C comments and literals with spaces while preserving offsets.""" + + chars = list(source) + index = 0 + state = "code" + quote = "" + while index < len(chars): + current = chars[index] + following = chars[index + 1] if index + 1 < len(chars) else "" + if state == "code": + if current == "/" and following == "/": + chars[index] = chars[index + 1] = " " + index += 2 + state = "line-comment" + continue + if current == "/" and following == "*": + chars[index] = chars[index + 1] = " " + index += 2 + state = "block-comment" + continue + if current in ('"', "'"): + quote = current + chars[index] = " " + index += 1 + state = "literal" + continue + index += 1 + continue + if state == "line-comment": + if current == "\n": + state = "code" + else: + chars[index] = " " + index += 1 + continue + if state == "block-comment": + if current == "*" and following == "/": + chars[index] = chars[index + 1] = " " + index += 2 + state = "code" + else: + if current != "\n": + chars[index] = " " + index += 1 + continue + if state == "literal": + if current == "\\" and following: + chars[index] = " " + if following != "\n": + chars[index + 1] = " " + index += 2 + continue + if current == quote: + chars[index] = " " + index += 1 + state = "code" + continue + if current != "\n": + chars[index] = " " + index += 1 + return "".join(chars) + + +def _scan_calls(masked: str) -> list[Call]: + calls: list[Call] = [] + for match in CALL_RE.finditer(masked): + opening = match.end() - 1 + depth = 0 + closing = len(masked) + for cursor in range(opening, len(masked)): + if masked[cursor] == "(": + depth += 1 + elif masked[cursor] == ")": + depth -= 1 + if depth == 0: + closing = cursor + 1 + break + raw = masked[match.start("name") : closing] + calls.append( + Call( + name=match.group("name"), + line=masked.count("\n", 0, match.start("name")) + 1, + start=match.start("name"), + end=closing, + text=re.sub(r"\s+", " ", raw).strip(), + ) + ) + return calls + + +def _name_before_body(masked: str, opening: int) -> str | None: + prefix = masked[max(0, opening - 4096) : opening] + match = re.search(r"([A-Za-z_][A-Za-z0-9_]*)\s*\([^;{}]*\)\s*$", prefix, re.DOTALL) + return match.group(1) if match else None + + +def _function_ranges(masked: str) -> list[FunctionRange]: + ranges: list[FunctionRange] = [] + depth = 0 + opening = -1 + function_name: str | None = None + for index, character in enumerate(masked): + if character == "{": + if depth == 0: + opening = index + function_name = _name_before_body(masked, index) + depth += 1 + elif character == "}" and depth > 0: + depth -= 1 + if depth == 0 and function_name is not None: + ranges.append(FunctionRange(function_name, opening, index + 1)) + opening = -1 + function_name = None + return ranges + + +def _scope_for(call: Call, ranges: Sequence[FunctionRange]) -> FunctionRange | None: + for scope in ranges: + if scope.body_start < call.start < scope.body_end: + return scope + return None + + +def _fullmatch(pattern: str, call: Call) -> bool: + return re.fullmatch(pattern, call.text) is not None + + +SOCKET_UNIX = r"socket\s*\(\s*AF_UNIX\s*,\s*SOCK_STREAM\s*,\s*0\s*\)" +SOCKET_LOOPBACK = r"socket\s*\(\s*AF_INET\s*,\s*SOCK_STREAM\s*,\s*0\s*\)" +CONNECT_UNIX = ( + r"connect\s*\(\s*fd\s*,\s*\(\s*const\s+struct\s+sockaddr\s*\*\s*\)\s*" + r"address\s*,\s*address_length\s*\)" +) +CONNECT_LOOPBACK = ( + r"connect\s*\(\s*socket_handle\s*,\s*\(\s*const\s+struct\s+sockaddr\s*\*\s*\)" + r"\s*&\s*address\s*,\s*sizeof\s*\(\s*address\s*\)\s*\)" +) + + +def _loopback_connect_witness(masked: str, call: Call, scope: FunctionRange) -> bool: + body = masked[scope.body_start + 1 : scope.body_end - 1] + call_offset = call.start - scope.body_start - 1 + # The complete witness must be one unconditional, contiguous statement + # sequence in the same helper. Merely finding the three assignments in + # order is insufficient: `if (false) address.sin_family = ...` must not + # satisfy a local-only guarantee. + witness = re.compile( + r"\bstruct\s+sockaddr_in\s+address\s*;\s*" + r"memset\s*\(\s*&\s*address\s*,\s*0\s*,\s*sizeof\s*\(\s*address\s*\)\s*\)\s*;\s*" + r"address\s*\.\s*sin_family\s*=\s*AF_INET\s*;\s*" + r"address\s*\.\s*sin_port\s*=\s*htons\s*\(\s*\(\s*unsigned\s+short\s*\)" + r"\s*port\s*\)\s*;\s*" + r"address\s*\.\s*sin_addr\s*\.\s*s_addr\s*=\s*htonl\s*\(\s*0x7F000001U\s*\)\s*;\s*" + r"if\s*\(\s*$", + re.DOTALL, + ) + return witness.search(body[:call_offset]) is not None + + +def _semantic_ok(masked: str, calls: Sequence[Call], allowance: Allowance) -> bool: + ranges = _function_ranges(masked) + scopes = [_scope_for(call, ranges) for call in calls] + if any(scope is None or scope.name != allowance.scope for scope in scopes): + return False + if allowance.transport == "unix" and allowance.function == "socket": + return all(_fullmatch(SOCKET_UNIX, call) for call in calls) + if allowance.transport == "unix" and allowance.function == "connect": + if not all(_fullmatch(CONNECT_UNIX, call) for call in calls): + return False + scope = scopes[0] if scopes else None + if scope is None: + return False + header = masked[max(0, scope.body_start - 512) : scope.body_start] + return ( + re.search(r"const\s+struct\s+sockaddr_un\s*\*\s*address", header) is not None + ) + if allowance.transport == "loopback-v4" and allowance.function == "socket": + return all(_fullmatch(SOCKET_LOOPBACK, call) for call in calls) + if allowance.transport == "loopback-v4" and allowance.function == "connect": + if len(calls) != 1 or not _fullmatch(CONNECT_LOOPBACK, calls[0]): + return False + scope = scopes[0] if scopes else None + return scope is not None and _loopback_connect_witness(masked, calls[0], scope) + return False + + +def _read_allowances(path: pathlib.Path) -> tuple[list[Allowance], list[str]]: + allowances: list[Allowance] = [] + errors: list[str] = [] + for number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not raw_line.startswith("NETWORK:"): + continue + fields = raw_line.split(":", 6) + if len(fields) != 7: + errors.append(f"{path}:{number}: malformed NETWORK allowance") + continue + _, file_name, function, expected_text, transport, scope, justification = fields + try: + expected = int(expected_text) + except ValueError: + errors.append(f"{path}:{number}: invalid expected count {expected_text!r}") + continue + if expected <= 0 or function not in RAW_FUNCTIONS or not scope or not justification: + errors.append(f"{path}:{number}: incomplete NETWORK allowance") + continue + allowances.append( + Allowance(file_name, function, expected, transport, scope, justification) + ) + return allowances, errors + + +def _audit(repo_root: pathlib.Path, allowlist_path: pathlib.Path) -> int: + allowances, errors = _read_allowances(allowlist_path) + by_key: dict[tuple[str, str], list[Allowance]] = {} + for allowance in allowances: + by_key.setdefault((allowance.file, allowance.function), []).append(allowance) + consumed: set[tuple[str, str]] = set() + for source_path in sorted((repo_root / "src").rglob("*.c")): + relative = source_path.relative_to(repo_root).as_posix() + if relative == "src/ui/httpd.c": + continue + masked = _mask_non_code(source_path.read_text(encoding="utf-8", errors="replace")) + all_calls = _scan_calls(masked) + for function in RAW_FUNCTIONS: + calls = [call for call in all_calls if call.name == function] + if not calls: + continue + key = (relative, function) + entries = by_key.get(key, []) + consumed.add(key) + if len(entries) != 1: + print(f"BLOCKED: {relative}: unexpected raw {function}() surface") + print(f" expected=none actual={len(calls)} transport=none semantic_ok=false") + for call in calls: + print(f" {call.line}: {call.text}") + errors.append(f"{relative}:{function}: allowance count is {len(entries)}") + continue + allowance = entries[0] + semantic_ok = _semantic_ok(masked, calls, allowance) + if len(calls) != allowance.expected or not semantic_ok: + print(f"BLOCKED: {relative}: unexpected raw {function}() surface") + print( + f" expected={allowance.expected} actual={len(calls)} " + f"transport={allowance.transport} scope={allowance.scope} " + f"semantic_ok={str(semantic_ok).lower()}" + ) + for call in calls: + print(f" {call.line}: {call.text}") + errors.append(f"{relative}:{function}: reviewed shape changed") + else: + print( + f"REVIEWED: {relative}: {len(calls)} count-bounded " + f"{allowance.transport} {function}() call(s) in {allowance.scope}" + ) + for key, entries in sorted(by_key.items()): + if key not in consumed: + errors.append(f"{key[0]}:{key[1]}: stale NETWORK allowance ({len(entries)} entry)") + for error in errors: + print(f" policy-error: {error}", file=sys.stderr) + return 1 if errors else 0 + + +class SelfTestFailure(RuntimeError): + pass + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise SelfTestFailure(message) + + +def _self_test() -> None: + scanner_fixture = r''' +// socket(AF_INET, SOCK_STREAM, 0); connect(fd, bad, 1); +const char *decoy = "socket(AF_INET, SOCK_STREAM, 0)"; +static void calls(void) { +socket (AF_INET6, SOCK_STREAM, 0); socket(AF_INET, SOCK_STREAM, 0); + /* connect(fd, bad, 1); */ connect (fd, bad, 1); +} +''' + scanned = _scan_calls(_mask_non_code(scanner_fixture)) + _require( + [call.name for call in scanned] == ["socket", "socket", "connect"], + "comments, literals, whitespace, or same-line calls were counted incorrectly", + ) + _require([call.line for call in scanned] == [5, 5, 6], "call line mapping changed") + + safe = r''' +static bool loopback_probe(main_daemon_ctl_socket_t socket_handle, int port) { + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = htons((unsigned short)port); + address.sin_addr.s_addr = htonl(0x7F000001U); + if (connect (socket_handle, (const struct sockaddr *)&address, sizeof(address)) == 0) { + return true; + } + return false; +} +static bool endpoint(void) { + return socket (AF_INET, SOCK_STREAM, 0) >= 0; +} +''' + masked = _mask_non_code(safe) + calls = _scan_calls(masked) + connect_allowance = Allowance( + "src/main.c", "connect", 1, "loopback-v4", "loopback_probe", "self-test" + ) + socket_allowance = Allowance( + "src/main.c", "socket", 1, "loopback-v4", "endpoint", "self-test" + ) + _require( + _semantic_ok(masked, [call for call in calls if call.name == "connect"], connect_allowance), + "safe loopback connect witness was rejected", + ) + _require( + _semantic_ok(masked, [call for call in calls if call.name == "socket"], socket_allowance), + "safe loopback socket witness was rejected", + ) + + non_loopback = safe.replace("0x7F000001U", "0x08080808U") + masked_bad = _mask_non_code(non_loopback) + _require( + not _semantic_ok( + masked_bad, + [call for call in _scan_calls(masked_bad) if call.name == "connect"], + connect_allowance, + ), + "non-loopback address was accepted", + ) + reassigned = safe.replace( + " if (connect", + " address.sin_addr.s_addr = htonl(0x08080808U);\n if (connect", + ) + masked_bad = _mask_non_code(reassigned) + _require( + not _semantic_ok( + masked_bad, + [call for call in _scan_calls(masked_bad) if call.name == "connect"], + connect_allowance, + ), + "post-witness address reassignment was accepted", + ) + conditional = safe.replace( + " address.sin_family = AF_INET;\n" + " address.sin_port = htons((unsigned short)port);\n" + " address.sin_addr.s_addr = htonl(0x7F000001U);", + " if (false) address.sin_family = AF_INET;\n" + " if (false) address.sin_port = htons((unsigned short)port);\n" + " if (false) address.sin_addr.s_addr = htonl(0x7F000001U);", + ) + masked_bad = _mask_non_code(conditional) + _require( + not _semantic_ok( + masked_bad, + [call for call in _scan_calls(masked_bad) if call.name == "connect"], + connect_allowance, + ), + "conditional loopback assignments were accepted", + ) + wrong_family = safe.replace("socket (AF_INET, SOCK_STREAM, 0)", + "/* socket(AF_INET, SOCK_STREAM, 0) */\n" + " socket (AF_INET6, SOCK_STREAM, 0)") + masked_bad = _mask_non_code(wrong_family) + _require( + not _semantic_ok( + masked_bad, + [call for call in _scan_calls(masked_bad) if call.name == "socket"], + socket_allowance, + ), + "wrong socket family plus comment decoy was accepted", + ) + + +def main(argv: Sequence[str]) -> int: + try: + _self_test() + except SelfTestFailure as error: + print(f"BLOCKED: network source audit self-test failed: {error}", file=sys.stderr) + return 1 + if len(argv) == 2 and argv[1] == "--self-test": + print("network source audit self-test: OK") + return 0 + if len(argv) != 3: + print(f"usage: {argv[0]} REPO_ROOT ALLOWLIST", file=sys.stderr) + return 2 + return _audit(pathlib.Path(argv[1]).resolve(), pathlib.Path(argv[2]).resolve()) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) From 00ed56fb77d728e658c7d61a97ff2d06642b88b2 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 18:52:29 +0200 Subject: [PATCH 03/13] fix(build): restore Windows release compile Signed-off-by: Martin Vogel --- Makefile.cbm | 2 ++ src/daemon/runtime.c | 1 + 2 files changed, 3 insertions(+) diff --git a/Makefile.cbm b/Makefile.cbm index a7c4b42b16..bc7a829fa3 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -164,8 +164,10 @@ endif # is meaningless for PE, so it is gated rather than made "common". ELF_HARDENING_FLAGS := ifeq ($(IS_LINUX),yes) +ifneq ($(IS_MINGW),yes) ELF_HARDENING_FLAGS := -Wl,-z,noexecstack endif +endif # The POSIX wrap shim exists only so the profiler can observe allocations. It # must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan, diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 1699ebfb59..bdbb2c56bb 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -11,6 +11,7 @@ #include "foundation/log.h" #include "foundation/mem.h" #include "foundation/platform.h" +#include "foundation/win_utf8.h" #include #include From 54b5ba846bdcfbac87aaef239b6c56665a6f5b49 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 19:18:47 +0200 Subject: [PATCH 04/13] fix(lint): generate integration hash before cppcheck Signed-off-by: Martin Vogel --- Makefile.cbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.cbm b/Makefile.cbm index bc7a829fa3..246e8f1d23 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -1114,7 +1114,7 @@ lint-tidy: @$(CLANG_TIDY) --quiet $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG) # cppcheck: complementary analysis (config in .cppcheck) -lint-cppcheck: +lint-cppcheck: $(INTEGRATIONS_HASH_HDR) @echo "=== cppcheck ===" @$(CPPCHECK) --enable=warning,style,performance,portability \ --std=c11 --language=c \ From 7a62a085cf8ec4cc624fa9d9b3fbe6a1c313d29e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 20:53:14 +0200 Subject: [PATCH 05/13] fix(packages): make wrapper locks Windows-safe Signed-off-by: Martin Vogel --- pkg/go/cmd/codebase-memory-mcp/main.go | 94 +++++- pkg/go/cmd/codebase-memory-mcp/main_test.go | 20 ++ .../runtime_lock_windows.go | 29 +- .../runtime_lock_windows_test.go | 66 +++++ pkg/npm/test/windows-launcher.test.js | 6 +- pkg/pypi/src/codebase_memory_mcp/_cli.py | 141 +++++++-- pkg/pypi/tests/test_cli.py | 279 +++++++++++++++++- scripts/ci/test-package-wrappers.sh | 0 8 files changed, 582 insertions(+), 53 deletions(-) create mode 100644 pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go mode change 100644 => 100755 scripts/ci/test-package-wrappers.sh diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index ffc8efdfdc..8abcc70a4b 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -1742,6 +1742,34 @@ func runtimeSetSameLockObject(left, right os.FileInfo) bool { left.Mode().Type() == right.Mode().Type() } +// captureRuntimeSetLockObject resolves the lock's file identity while its +// canonical pathname still exists. On Windows, os.FileInfo identity can be +// resolved lazily from its pathname; carrying an untouched Lstat result across +// a rename therefore loses the object we meant to revalidate. +func captureRuntimeSetLockObject( + lockPath string, observedStatus os.FileInfo, +) (os.FileInfo, error) { + object, err := os.Open(lockPath) + if err != nil { + return nil, err + } + descriptorStatus, descriptorErr := object.Stat() + canonicalStatus, canonicalErr := os.Lstat(lockPath) + identityOK := descriptorErr == nil && canonicalErr == nil && + runtimeSetSameLockObject(observedStatus, descriptorStatus) && + runtimeSetSameLockObject(descriptorStatus, canonicalStatus) + closeErr := object.Close() + if !identityOK { + return nil, fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) + } + if closeErr != nil { + return nil, closeErr + } + return descriptorStatus, nil +} + func writeRuntimeSetLockRecord(owner *os.File, token string) error { if err := owner.Truncate(0); err != nil { return err @@ -1813,6 +1841,10 @@ func runtimeSetTryReclaimLock(lockPath, contenderToken string) bool { (!observedStatus.Mode().IsRegular() && !observedStatus.IsDir()) { return false } + observedStatus, err = captureRuntimeSetLockObject(lockPath, observedStatus) + if err != nil { + return os.IsNotExist(err) + } owner, ownerOK := runtimeSetLockOwner(lockPath) if !runtimeSetOwnerReclaimable(observedStatus, owner, ownerOK) { return false @@ -1893,16 +1925,37 @@ func acquireRuntimeSetLock(destinationDirectory string) (*runtimeSetLock, error) return nil, err } } + descriptorStatus, descriptorErr := owner.Stat() + claimStatus, claimStatusErr := os.Lstat(claimPath) + if descriptorErr != nil || claimStatusErr != nil || + !runtimeSetSameLockObject(descriptorStatus, claimStatus) { + _ = owner.Close() + _ = os.Remove(claimPath) + return nil, fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) + } + if err := owner.Close(); err != nil { + _ = os.Remove(claimPath) + return nil, err + } linkErr := os.Link(claimPath, lockPath) if linkErr == nil { if err := os.Remove(claimPath); err != nil { - _ = owner.Close() return nil, err } - descriptorStatus, descriptorErr := owner.Stat() + owner, openErr := os.OpenFile(lockPath, os.O_RDWR, 0) + if openErr != nil { + return nil, openErr + } + reopenedStatus, reopenedErr := owner.Stat() canonicalStatus, canonicalErr := os.Lstat(lockPath) - if descriptorErr != nil || canonicalErr != nil || - !runtimeSetSameLockObject(descriptorStatus, canonicalStatus) { + lockOwner, ownerOK := runtimeSetLockOwner(lockPath) + if reopenedErr != nil || canonicalErr != nil || + !runtimeSetSameLockObject(descriptorStatus, reopenedStatus) || + !runtimeSetSameLockObject(reopenedStatus, canonicalStatus) || + !ownerOK || lockOwner.PID != os.Getpid() || + lockOwner.Token != token { _ = owner.Close() return nil, fmt.Errorf( "package-cache runtime-set publication lock ownership changed", @@ -1910,7 +1963,6 @@ func acquireRuntimeSetLock(destinationDirectory string) (*runtimeSetLock, error) } return &runtimeSetLock{path: lockPath, token: token, file: owner}, nil } - _ = owner.Close() _ = os.Remove(claimPath) if !os.IsExist(linkErr) { return nil, linkErr @@ -1934,32 +1986,42 @@ func releaseRuntimeSetLock(lock *runtimeSetLock) error { if err := assertRuntimeSetLockOwner(lock); err != nil { if lock != nil && lock.file != nil { _ = lock.file.Close() + lock.file = nil } return err } + descriptorStatus, descriptorErr := lock.file.Stat() + canonicalStatus, canonicalErr := os.Lstat(lock.path) + owner, ownerOK := runtimeSetLockOwner(lock.path) + if descriptorErr != nil || canonicalErr != nil || + !runtimeSetSameLockObject(descriptorStatus, canonicalStatus) || + !ownerOK || owner.PID != os.Getpid() || owner.Token != lock.token { + _ = lock.file.Close() + lock.file = nil + return fmt.Errorf( + "package-cache runtime-set publication lock ownership changed", + ) + } + if err := lock.file.Close(); err != nil { + lock.file = nil + return err + } + lock.file = nil released := lock.path + ".released-" + lock.token if err := os.Rename(lock.path, released); err != nil { - _ = lock.file.Close() return err } - descriptorStatus, descriptorErr := lock.file.Stat() releasedStatus, releasedErr := os.Lstat(released) - owner, ownerOK := runtimeSetLockOwner(released) - if descriptorErr != nil || releasedErr != nil || + owner, ownerOK = runtimeSetLockOwner(released) + if releasedErr != nil || !runtimeSetSameLockObject(descriptorStatus, releasedStatus) || !ownerOK || owner.PID != os.Getpid() || owner.Token != lock.token { restoreDisplacedRuntimeSetLock(released, lock.path, releasedStatus) - _ = lock.file.Close() return fmt.Errorf( "package-cache runtime-set publication lock ownership changed", ) } - removeErr := os.Remove(released) - closeErr := lock.file.Close() - if removeErr != nil { - return removeErr - } - return closeErr + return os.Remove(released) } func attachRuntimeLockReleaseError(result *error, releaseErr error) { diff --git a/pkg/go/cmd/codebase-memory-mcp/main_test.go b/pkg/go/cmd/codebase-memory-mcp/main_test.go index c362e8f65f..c014484f4b 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/main_test.go @@ -1930,6 +1930,26 @@ func TestStalledRuntimeLockCreatorNeverDeletesSuccessor(t *testing.T) { } } +func TestRuntimeSetLockReleaseRetiresDescriptor(t *testing.T) { + destination := t.TempDir() + lock, err := acquireRuntimeSetLock(destination) + if err != nil { + t.Fatal(err) + } + if lock.file == nil { + t.Fatal("acquired runtime-set lock has no writable descriptor") + } + if err := releaseRuntimeSetLock(lock); err != nil { + t.Fatal(err) + } + if lock.file != nil { + t.Fatal("released runtime-set lock retained its closed descriptor") + } + if _, err := os.Lstat(lock.path); !os.IsNotExist(err) { + t.Fatalf("released runtime-set lock remained at its canonical path: %v", err) + } +} + func TestRuntimeReadinessRejectsMultiplyLinkedLeaves(t *testing.T) { directory := t.TempDir() binary := "codebase-memory-mcp" diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go index 3fa233c5a2..5b1f7f89b2 100644 --- a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go @@ -166,16 +166,29 @@ func platformRuntimeSetLockProcessAlive(pid int) bool { // evidence that an otherwise valid owner is dead. return true } - // PROCESS_QUERY_LIMITED_INFORMATION is sufficient for an existence probe and - // is available on every Windows version supported by the package wrapper. - handle, err := syscall.OpenProcess(0x1000, false, uint32(pid)) - if err == nil { - _ = syscall.CloseHandle(handle) + // A process object can remain open after its process exits. SYNCHRONIZE lets + // us distinguish that signaled object from a running process instead of + // treating every successful OpenProcess call as proof that the PID is live. + handle, err := syscall.OpenProcess(syscall.SYNCHRONIZE, false, uint32(pid)) + if err != nil { + // ERROR_INVALID_PARAMETER is the documented result for a process + // identifier whose object no longer exists. Access denial and every other + // probe error are conservatively treated as live. + return err != syscall.Errno(87) + } + event, waitErr := syscall.WaitForSingleObject(handle, 0) + closeErr := syscall.CloseHandle(handle) + if waitErr != nil || closeErr != nil { + return true + } + switch event { + case syscall.WAIT_OBJECT_0: + return false + case syscall.WAIT_TIMEOUT: + return true + default: return true } - // ERROR_INVALID_PARAMETER is the documented result for a process identifier - // that no longer exists. Access denial is conservatively treated as live. - return err != syscall.Errno(87) } func platformRuntimeSetFileLinkCountOne(path string, _ os.FileInfo) bool { diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go new file mode 100644 index 0000000000..190210465e --- /dev/null +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go @@ -0,0 +1,66 @@ +//go:build windows + +package main + +import ( + "os" + "os/exec" + "syscall" + "testing" +) + +const windowsExitedProcessProbeHelper = "CBM_GO_EXITED_PROCESS_PROBE_HELPER" + +func TestWindowsExitedProcessProbeHelper(t *testing.T) { + if os.Getenv(windowsExitedProcessProbeHelper) != "1" { + t.Skip("helper process only") + } +} + +func TestRuntimeSetProcessProbeRejectsExitedOpenProcess(t *testing.T) { + command := exec.Command( + os.Args[0], "-test.run=^TestWindowsExitedProcessProbeHelper$", + ) + command.Env = append(os.Environ(), windowsExitedProcessProbeHelper+"=1") + if err := command.Start(); err != nil { + t.Fatal(err) + } + waited := false + defer func() { + if !waited { + _ = command.Process.Kill() + _ = command.Wait() + } + }() + + handle, err := syscall.OpenProcess( + syscall.SYNCHRONIZE, false, uint32(command.Process.Pid), + ) + if err != nil { + t.Fatal(err) + } + defer syscall.CloseHandle(handle) + event, err := syscall.WaitForSingleObject(handle, 5000) + if err != nil { + t.Fatal(err) + } + if event != syscall.WAIT_OBJECT_0 { + t.Fatalf("helper process wait result = %d, want WAIT_OBJECT_0", event) + } + + // Keep both the exec.Cmd handle and our explicit handle open. OpenProcess can + // still succeed in this state even though the process has already exited. + if platformRuntimeSetLockProcessAlive(command.Process.Pid) { + t.Fatal("exited process with a retained process object was reported live") + } + if err := command.Wait(); err != nil { + t.Fatal(err) + } + waited = true +} + +func TestRuntimeSetProcessProbeAcceptsCurrentProcess(t *testing.T) { + if !platformRuntimeSetLockProcessAlive(os.Getpid()) { + t.Fatal("current process was reported dead") + } +} diff --git a/pkg/npm/test/windows-launcher.test.js b/pkg/npm/test/windows-launcher.test.js index da19f459b6..6a63f58a12 100644 --- a/pkg/npm/test/windows-launcher.test.js +++ b/pkg/npm/test/windows-launcher.test.js @@ -205,7 +205,11 @@ test('wrapper uninstall locks the cache and targets the managed install by defau Array.from(observed.calls[1].args), [ 'uninstall', '--yes', '--dir', - 'C:\\Users\\test\\AppData\\Local/Programs/codebase-memory-mcp', + path.join( + 'C:\\Users\\test\\AppData\\Local', + 'Programs', + 'codebase-memory-mcp', + ), ], ); }); diff --git a/pkg/pypi/src/codebase_memory_mcp/_cli.py b/pkg/pypi/src/codebase_memory_mcp/_cli.py index 05f854e6f4..cf15387925 100644 --- a/pkg/pypi/src/codebase_memory_mcp/_cli.py +++ b/pkg/pypi/src/codebase_memory_mcp/_cli.py @@ -58,6 +58,10 @@ _RUNTIME_BACKUP_RE = re.compile(r"\.cbm-runtime-backup-[0-9a-f]{32}\Z") _RUNTIME_BACKUP_RETIRED_MARKER = ".retirement-complete" _RUNTIME_BACKUP_CLEANUP_MARKER = ".cleanup-only" +_WINDOWS_SYNCHRONIZE = 0x00100000 +_WINDOWS_WAIT_OBJECT_0 = 0x00000000 +_WINDOWS_WAIT_TIMEOUT = 0x00000102 +_WINDOWS_ERROR_INVALID_PARAMETER = 87 _runtime_lock_claim_observer = None @@ -544,9 +548,67 @@ def _staged_runtime_names(staged_paths, binary_name: str, variant: str): return runtime_names +def _windows_process_api(): + import ctypes + from ctypes import wintypes + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.OpenProcess.argtypes = ( + wintypes.DWORD, + wintypes.BOOL, + wintypes.DWORD, + ) + kernel32.OpenProcess.restype = wintypes.HANDLE + kernel32.WaitForSingleObject.argtypes = ( + wintypes.HANDLE, + wintypes.DWORD, + ) + kernel32.WaitForSingleObject.restype = wintypes.DWORD + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + kernel32.CloseHandle.restype = wintypes.BOOL + return kernel32, ctypes.get_last_error + + +def _windows_process_is_alive(pid: int) -> bool: + if pid > 0xFFFFFFFF: + # Avoid truncating an unprobeable owner identifier into another PID. + return True + try: + kernel32, get_last_error = _windows_process_api() + handle = kernel32.OpenProcess(_WINDOWS_SYNCHRONIZE, False, pid) + except Exception: + return True + if not handle: + try: + error = get_last_error() + except Exception: + return True + # OpenProcess documents ERROR_INVALID_PARAMETER for a PID that no + # longer identifies a process. Access denial remains evidence of a + # potentially live owner and must not authorize lock reclamation. + return error != _WINDOWS_ERROR_INVALID_PARAMETER + try: + try: + wait_result = kernel32.WaitForSingleObject(handle, 0) + except Exception: + return True + if wait_result == _WINDOWS_WAIT_OBJECT_0: + return False + if wait_result == _WINDOWS_WAIT_TIMEOUT: + return True + return True + finally: + try: + kernel32.CloseHandle(handle) + except Exception: + pass + + def _process_is_alive(pid: int) -> bool: if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: return False + if os.name == "nt": + return _windows_process_is_alive(pid) try: os.kill(pid, 0) return True @@ -699,6 +761,35 @@ def _assert_runtime_lock_owner(lock): raise RuntimeError("package-cache publication lock ownership changed") +def _open_validated_runtime_lock(lock_path: Path, token: str, expected_status): + """Reopen a lock name and prove it still names the expected owner object.""" + try: + owner_fd = os.open(lock_path, os.O_RDWR) + except OSError as exc: + raise RuntimeError( + "package-cache publication lock ownership changed" + ) from exc + try: + descriptor_status = os.fstat(owner_fd) + canonical_status = _runtime_lock_status(lock_path) + owner = _read_runtime_lock_owner(lock_path) + if ( + not _same_runtime_lock_object(expected_status, descriptor_status) + or not _same_runtime_lock_object( + descriptor_status, canonical_status + ) + or owner is None + or owner[:2] != (os.getpid(), token) + ): + raise RuntimeError( + "package-cache publication lock ownership changed" + ) + except BaseException: + os.close(owner_fd) + raise + return owner_fd + + def _refresh_runtime_lock(lock): _assert_runtime_lock_owner(lock) _write_runtime_lock_owner(lock[2], lock[1]) @@ -724,16 +815,26 @@ def _acquire_runtime_lock(directory: Path): if _runtime_lock_claim_observer is not None: _runtime_lock_claim_observer() os.link(claim_path, lock_path) - claim_path.unlink() + linked_status = os.fstat(owner_fd) if not _same_runtime_lock_object( - os.fstat(owner_fd), _runtime_lock_status(lock_path) + linked_status, _runtime_lock_status(lock_path) ): raise RuntimeError( "package-cache publication lock ownership changed" ) - except BaseException: + # Windows does not grant delete sharing to this descriptor. + # Close it before removing the claim name, then reopen the + # canonical name and revalidate identity plus owner record. os.close(owner_fd) owner_fd = None + claim_path.unlink() + owner_fd = _open_validated_runtime_lock( + lock_path, token, linked_status + ) + except BaseException: + if owner_fd is not None: + os.close(owner_fd) + owner_fd = None try: claim_path.unlink() except OSError: @@ -762,27 +863,27 @@ def _release_runtime_lock(lock): except Exception: os.close(owner_fd) raise + descriptor_status = os.fstat(owner_fd) + # Path mutation while this descriptor is open fails on Windows. Close it, + # then reopen the canonical name to prove no object or owner substitution + # occurred before releasing the name. + os.close(owner_fd) + validated_fd = _open_validated_runtime_lock( + lock_path, token, descriptor_status + ) + os.close(validated_fd) released = lock_path.with_name(f"{lock_path.name}.released-{token}") + os.rename(lock_path, released) + released_status = _runtime_lock_status(released) try: - os.rename(lock_path, released) + validated_fd = _open_validated_runtime_lock( + released, token, descriptor_status + ) except Exception: - os.close(owner_fd) - raise - descriptor_status = os.fstat(owner_fd) - released_status = _runtime_lock_status(released) - owner = _read_runtime_lock_owner(released) - if ( - not _same_runtime_lock_object(descriptor_status, released_status) - or owner is None - or owner[:2] != (os.getpid(), token) - ): _restore_displaced_runtime_lock(released, lock_path, released_status) - os.close(owner_fd) - raise RuntimeError("package-cache publication lock ownership changed") - try: - released.unlink() - finally: - os.close(owner_fd) + raise + os.close(validated_fd) + released.unlink() # A grouped backup is a small crash journal. The retired marker separates a diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index 8590280b29..6d927fefa6 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -1,3 +1,4 @@ +import errno import hashlib import json import os @@ -46,15 +47,45 @@ def test_cache_sensitive_mutation_classification_is_explicit(self): self.assertIsNone(_cli._runtime_mutation_action(["daemon", "update"])) def test_wrapper_uninstall_never_defaults_to_its_cache_binary(self): - with mock.patch.object(Path, "home", return_value=Path("/home/user")): - self.assertEqual( - _cli._native_args(["uninstall", "--yes"]), - ["uninstall", "--yes", "--dir", "/home/user/.local/bin"], - ) - self.assertEqual( - _cli._native_args(["uninstall", "--dir", "/custom/bin"]), - ["uninstall", "--dir", "/custom/bin"], + linux_home = Path("fixtures") / "linux-home" + windows_local_app_data = Path("fixtures") / "windows-local-app-data" + defaults = ( + ( + "linux", + linux_home / ".local" / "bin", + mock.patch.object(Path, "home", return_value=linux_home), + ), + ( + "win32", + windows_local_app_data + / "Programs" + / "codebase-memory-mcp", + mock.patch.dict( + os.environ, + {"LOCALAPPDATA": str(windows_local_app_data)}, + ), + ), ) + for platform_name, expected, path_fixture in defaults: + with self.subTest(platform=platform_name), mock.patch.object( + _cli.sys, "platform", platform_name + ), path_fixture: + self.assertEqual( + _cli._native_args(["uninstall", "--yes"]), + ["uninstall", "--yes", "--dir", str(expected)], + ) + + explicit_dir = Path("fixtures") / "explicit-install" + for platform_name in ("linux", "win32"): + with self.subTest( + platform=platform_name, explicit_dir=True + ), mock.patch.object(_cli.sys, "platform", platform_name): + self.assertEqual( + _cli._native_args( + ["uninstall", "--dir", str(explicit_dir)] + ), + ["uninstall", "--dir", str(explicit_dir)], + ) def test_standard_and_ui_cache_paths_do_not_collide(self): with mock.patch.dict("os.environ", {"CBM_VARIANT": "standard"}): @@ -82,6 +113,99 @@ def test_update_guidance_uses_pip_and_names_managed_install(self): self.assertNotIn("install.sh", output) +class ProcessLivenessTests(unittest.TestCase): + def test_windows_probe_never_calls_os_kill(self): + with mock.patch.object(_cli.os, "name", "nt"), mock.patch.object( + _cli, + "_windows_process_is_alive", + return_value=True, + create=True, + ) as windows_probe, mock.patch.object( + _cli.os, + "kill", + side_effect=AssertionError("Windows must not call os.kill"), + ): + self.assertTrue(_cli._process_is_alive(1234)) + + windows_probe.assert_called_once_with(1234) + + def test_windows_wait_result_distinguishes_dead_and_live_processes(self): + cases = ( + (_cli._WINDOWS_WAIT_OBJECT_0, False), + (_cli._WINDOWS_WAIT_TIMEOUT, True), + (0xFFFFFFFF, True), + ) + for wait_result, expected in cases: + with self.subTest(wait_result=wait_result): + kernel32 = mock.Mock() + kernel32.OpenProcess.return_value = 123 + kernel32.WaitForSingleObject.return_value = wait_result + with mock.patch.object( + _cli, + "_windows_process_api", + return_value=(kernel32, mock.Mock()), + ): + self.assertEqual( + _cli._windows_process_is_alive(4567), expected + ) + + kernel32.OpenProcess.assert_called_once_with( + _cli._WINDOWS_SYNCHRONIZE, False, 4567 + ) + kernel32.WaitForSingleObject.assert_called_once_with(123, 0) + kernel32.CloseHandle.assert_called_once_with(123) + + def test_windows_open_failure_only_treats_missing_pid_as_dead(self): + cases = ( + (_cli._WINDOWS_ERROR_INVALID_PARAMETER, False), + (5, True), + ) + for error, expected in cases: + with self.subTest(error=error): + kernel32 = mock.Mock() + kernel32.OpenProcess.return_value = 0 + get_last_error = mock.Mock(return_value=error) + with mock.patch.object( + _cli, + "_windows_process_api", + return_value=(kernel32, get_last_error), + ): + self.assertEqual( + _cli._windows_process_is_alive(4567), expected + ) + + get_last_error.assert_called_once_with() + kernel32.WaitForSingleObject.assert_not_called() + kernel32.CloseHandle.assert_not_called() + + def test_windows_api_failures_are_conservatively_live(self): + kernel32 = mock.Mock() + kernel32.OpenProcess.side_effect = OSError("injected API failure") + with mock.patch.object( + _cli, + "_windows_process_api", + return_value=(kernel32, mock.Mock()), + ): + self.assertTrue(_cli._windows_process_is_alive(4567)) + + def test_posix_probe_behavior_is_unchanged(self): + cases = ( + (None, True), + (ProcessLookupError(), False), + (PermissionError(), True), + (OSError(errno.ESRCH, "missing"), False), + ) + for error, expected in cases: + with self.subTest(error=type(error).__name__): + side_effect = error if error is not None else None + with mock.patch.object(_cli.os, "name", "posix"), \ + mock.patch.object( + _cli.os, "kill", side_effect=side_effect + ) as kill: + self.assertEqual(_cli._process_is_alive(4567), expected) + kill.assert_called_once_with(4567, 0) + + class RuntimeSetTests(unittest.TestCase): def test_candidate_probe_authenticates_adjacent_runtime_assets(self): with mock.patch.object(_cli.subprocess, "run") as run: @@ -151,6 +275,145 @@ def publish_successor_then_abort(): self.assertTrue(successor[0].exists()) _cli._release_runtime_lock(successor) + def test_acquire_closes_claim_descriptor_before_unlink_and_reopens(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + captured = {} + write_owner = _cli._write_runtime_lock_owner + unlink = Path.unlink + + def capture_claim_descriptor(owner_fd, token): + captured["claim_fd"] = owner_fd + write_owner(owner_fd, token) + + def require_closed_claim_descriptor(path, *args, **kwargs): + if f"{_cli._RUNTIME_LOCK_NAME}.claim-" in path.name: + with self.assertRaises(OSError): + os.fstat(captured["claim_fd"]) + captured["closed_before_unlink"] = True + return unlink(path, *args, **kwargs) + + with mock.patch.object( + _cli, + "_write_runtime_lock_owner", + side_effect=capture_claim_descriptor, + ), mock.patch.object( + Path, "unlink", new=require_closed_claim_descriptor + ): + lock, _ = _cli._acquire_runtime_lock(directory) + + try: + self.assertTrue(captured["closed_before_unlink"]) + self.assertTrue( + _cli._same_runtime_lock_object( + os.fstat(lock[2]), lock[0].lstat() + ) + ) + finally: + _cli._release_runtime_lock(lock) + + def test_release_closes_descriptor_and_revalidates_each_lock_name(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + lock, _ = _cli._acquire_runtime_lock(directory) + original_fd = lock[2] + events = [] + read_owner = _cli._read_runtime_lock_owner + rename = os.rename + + def record_owner_validation(path): + events.append(("validate", Path(path).name)) + return read_owner(path) + + def require_closed_descriptor(source, destination): + with self.assertRaises(OSError): + os.fstat(original_fd) + events.append(("rename", Path(source).name)) + return rename(source, destination) + + with mock.patch.object( + _cli, + "_read_runtime_lock_owner", + side_effect=record_owner_validation, + ), mock.patch.object( + _cli.os, "rename", side_effect=require_closed_descriptor + ): + _cli._release_runtime_lock(lock) + + self.assertEqual( + events, + [ + ("validate", _cli._RUNTIME_LOCK_NAME), + ("validate", _cli._RUNTIME_LOCK_NAME), + ("rename", _cli._RUNTIME_LOCK_NAME), + ( + "validate", + f"{_cli._RUNTIME_LOCK_NAME}.released-{lock[1]}", + ), + ], + ) + self.assertFalse(lock[0].exists()) + + def test_release_rejects_owner_substitution_before_rename(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + lock, _ = _cli._acquire_runtime_lock(directory) + read_owner = _cli._read_runtime_lock_owner + canonical_reads = 0 + + def substitute_owner_on_reopen(path): + nonlocal canonical_reads + owner = read_owner(path) + if Path(path) == lock[0]: + canonical_reads += 1 + if canonical_reads == 2: + return (os.getpid(), "f" * 32, owner[2]) + return owner + + with mock.patch.object( + _cli, + "_read_runtime_lock_owner", + side_effect=substitute_owner_on_reopen, + ): + with self.assertRaisesRegex(RuntimeError, "ownership changed"): + _cli._release_runtime_lock(lock) + + self.assertTrue(lock[0].exists()) + lock[0].unlink() + + def test_release_rejects_object_substitution_before_rename(self): + with tempfile.TemporaryDirectory() as root: + directory = Path(root) + lock, _ = _cli._acquire_runtime_lock(directory) + close = os.close + substituted = False + + def substitute_object_after_owner_close(owner_fd): + nonlocal substituted + close(owner_fd) + if ( + owner_fd != lock[2] + or substituted + or not lock[0].exists() + ): + return + substituted = True + owner_record = lock[0].read_text(encoding="utf-8") + lock[0].unlink() + lock[0].write_text(owner_record, encoding="utf-8") + + with mock.patch.object( + _cli.os, + "close", + side_effect=substitute_object_after_owner_close, + ): + with self.assertRaisesRegex(RuntimeError, "ownership changed"): + _cli._release_runtime_lock(lock) + + self.assertTrue(substituted) + self.assertTrue(lock[0].exists()) + lock[0].unlink() + def test_expired_lease_never_reclaims_live_owner(self): with tempfile.TemporaryDirectory() as root: lock_path = Path(root) / _cli._RUNTIME_LOCK_NAME diff --git a/scripts/ci/test-package-wrappers.sh b/scripts/ci/test-package-wrappers.sh old mode 100644 new mode 100755 From a1a5b7fdb41b3c38fc4b89b755349abf9a1a6590 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 21:45:33 +0200 Subject: [PATCH 06/13] fix(packages): eliminate wrapper lock sharing races Signed-off-by: Martin Vogel --- pkg/go/cmd/codebase-memory-mcp/main.go | 35 ++++++++++---- pkg/go/cmd/codebase-memory-mcp/main_test.go | 47 +++++++++++++++++-- .../codebase-memory-mcp/runtime_lock_other.go | 4 ++ .../codebase-memory-mcp/runtime_lock_unix.go | 4 ++ .../runtime_lock_windows.go | 29 ++++++++++++ .../runtime_lock_windows_test.go | 42 +++++++++++++++++ pkg/pypi/tests/test_cli.py | 15 ++++-- 7 files changed, 158 insertions(+), 18 deletions(-) diff --git a/pkg/go/cmd/codebase-memory-mcp/main.go b/pkg/go/cmd/codebase-memory-mcp/main.go index 8abcc70a4b..2e7432c87b 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main.go +++ b/pkg/go/cmd/codebase-memory-mcp/main.go @@ -109,11 +109,13 @@ type runtimeBackupTransaction struct { // lock from one entering the shared publication sequence. Production leaves // it nil. var ( - runtimeSetLockWaitObserver func() - runtimeSetLockClaimObserver func() error - runtimeBackupCrashObserver func(string, string) error - runtimeSetProcessAlive = platformRuntimeSetLockProcessAlive - runtimeMutationSnapshotCleanup = os.RemoveAll + runtimeSetLockWaitObserver func() + runtimeSetLockClaimObserver func() error + runtimeSetLockCaptureObserver func() + runtimeSetLockOwnerReadObserver func() + runtimeBackupCrashObserver func(string, string) error + runtimeSetProcessAlive = platformRuntimeSetLockProcessAlive + runtimeMutationSnapshotCleanup = os.RemoveAll ) func main() { @@ -1721,10 +1723,18 @@ func runtimeSetLockOwner(lockPath string) (runtimeSetLockOwnerRecord, bool) { if err != nil || !ownerStatus.Mode().IsRegular() { return empty, false } - data, err := os.ReadFile(ownerPath) + ownerFile, err := platformOpenRuntimeSetLockOwner(ownerPath) if err != nil { return empty, false } + if runtimeSetLockOwnerReadObserver != nil { + runtimeSetLockOwnerReadObserver() + } + data, readErr := io.ReadAll(ownerFile) + closeErr := ownerFile.Close() + if readErr != nil || closeErr != nil { + return empty, false + } var record runtimeSetLockOwnerRecord if err := json.Unmarshal(data, &record); err != nil { return empty, false @@ -1753,6 +1763,9 @@ func captureRuntimeSetLockObject( if err != nil { return nil, err } + if runtimeSetLockCaptureObserver != nil { + runtimeSetLockCaptureObserver() + } descriptorStatus, descriptorErr := object.Stat() canonicalStatus, canonicalErr := os.Lstat(lockPath) identityOK := descriptorErr == nil && canonicalErr == nil && @@ -1841,14 +1854,16 @@ func runtimeSetTryReclaimLock(lockPath, contenderToken string) bool { (!observedStatus.Mode().IsRegular() && !observedStatus.IsDir()) { return false } - observedStatus, err = captureRuntimeSetLockObject(lockPath, observedStatus) - if err != nil { - return os.IsNotExist(err) - } owner, ownerOK := runtimeSetLockOwner(lockPath) if !runtimeSetOwnerReclaimable(observedStatus, owner, ownerOK) { return false } + // Pin identity only after the observed owner is reclaimable. On Windows, + // opening a proven-live lock here can deny the owner's release rename. + observedStatus, err = captureRuntimeSetLockObject(lockPath, observedStatus) + if err != nil { + return os.IsNotExist(err) + } reclaimed := lockPath + ".reclaimed-" + contenderToken if err := os.Rename(lockPath, reclaimed); err != nil { return os.IsNotExist(err) diff --git a/pkg/go/cmd/codebase-memory-mcp/main_test.go b/pkg/go/cmd/codebase-memory-mcp/main_test.go index c014484f4b..514a7f8a37 100644 --- a/pkg/go/cmd/codebase-memory-mcp/main_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/main_test.go @@ -1571,9 +1571,12 @@ func TestMutationSnapshotParentDeathLeavesCacheRecoverable(t *testing.T) { t.Fatal(err) } info := fmt.Sprintf("%d\n%s\n", os.Getpid(), executable) - if err := os.WriteFile( - os.Getenv("CBM_GO_MUTATION_CHILD_INFO"), []byte(info), 0600, - ); err != nil { + childInfo := os.Getenv("CBM_GO_MUTATION_CHILD_INFO") + stagedChildInfo := childInfo + ".tmp" + if err := os.WriteFile(stagedChildInfo, []byte(info), 0600); err != nil { + t.Fatal(err) + } + if err := os.Rename(stagedChildInfo, childInfo); err != nil { t.Fatal(err) } continuePath := os.Getenv("CBM_GO_MUTATION_CHILD_CONTINUE") @@ -1809,6 +1812,44 @@ func TestRuntimeSetLockReclaimsOnlyDefinitelyDeadOwner(t *testing.T) { } } +func TestRuntimeSetLiveOwnerSkipsIdentityCapture(t *testing.T) { + destination := t.TempDir() + lock, err := acquireRuntimeSetLock(destination) + if err != nil { + t.Fatal(err) + } + released := false + defer func() { + if !released { + if err := releaseRuntimeSetLock(lock); err != nil { + t.Errorf("release live runtime-set lock: %v", err) + } + } + }() + + priorObserver := runtimeSetLockCaptureObserver + priorProcessAlive := runtimeSetProcessAlive + defer func() { + runtimeSetLockCaptureObserver = priorObserver + runtimeSetProcessAlive = priorProcessAlive + }() + captures := 0 + runtimeSetLockCaptureObserver = func() { captures++ } + runtimeSetProcessAlive = func(int) bool { return true } + + contenderToken := strings.Repeat("b", runtimeSetLockTokenSize*2) + if runtimeSetTryReclaimLock(lock.path, contenderToken) { + t.Fatal("runtime-set lock reclaimed a proven-live owner") + } + if captures != 0 { + t.Fatalf("live runtime-set lock identity was captured %d times", captures) + } + if err := releaseRuntimeSetLock(lock); err != nil { + t.Fatalf("release live runtime-set lock: %v", err) + } + released = true +} + func TestRuntimeSetLockReclaimsOnlyStaleOwnerlessDirectory(t *testing.T) { destination := t.TempDir() lockPath := filepath.Join(destination, runtimeSetLockName) diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go index d4c8645476..eabe5ca8c2 100644 --- a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_other.go @@ -7,6 +7,10 @@ import ( "os" ) +func platformOpenRuntimeSetLockOwner(path string) (*os.File, error) { + return os.Open(path) +} + func platformCreatePrivateMutationRuntimeDirectory() (string, error) { directory, err := os.MkdirTemp("", "codebase-memory-mcp-mutation-*") if err != nil { diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go index 0c8158b2d4..9685f893be 100644 --- a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_unix.go @@ -8,6 +8,10 @@ import ( "syscall" ) +func platformOpenRuntimeSetLockOwner(path string) (*os.File, error) { + return os.Open(path) +} + func platformCreatePrivateMutationRuntimeDirectory() (string, error) { directory, err := os.MkdirTemp("", "codebase-memory-mcp-mutation-*") if err != nil { diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go index 5b1f7f89b2..14a72d3575 100644 --- a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows.go @@ -41,6 +41,35 @@ func windowsExtendedPath(path string) (string, error) { return `\\?\` + absolute, nil } +func platformOpenRuntimeSetLockOwner(path string) (*os.File, error) { + extended, err := windowsExtendedPath(path) + if err != nil { + return nil, err + } + pointer, err := syscall.UTF16PtrFromString(extended) + if err != nil { + return nil, err + } + handle, err := syscall.CreateFile( + pointer, + syscall.GENERIC_READ, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(handle), path) + if file == nil { + _ = syscall.CloseHandle(handle) + return nil, syscall.EINVAL + } + return file, nil +} + func windowsPrivateDirectorySecurityDescriptor() (uintptr, error) { token, err := syscall.OpenCurrentProcessToken() if err != nil { diff --git a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go index 190210465e..e3cc92fb37 100644 --- a/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go +++ b/pkg/go/cmd/codebase-memory-mcp/runtime_lock_windows_test.go @@ -5,6 +5,8 @@ package main import ( "os" "os/exec" + "strings" + "sync/atomic" "syscall" "testing" ) @@ -64,3 +66,43 @@ func TestRuntimeSetProcessProbeAcceptsCurrentProcess(t *testing.T) { t.Fatal("current process was reported dead") } } + +func TestRuntimeSetLiveOwnerReadAllowsReleaseRename(t *testing.T) { + lock, err := acquireRuntimeSetLock(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer func() { + if lock.file != nil { + _ = releaseRuntimeSetLock(lock) + } + }() + + priorObserver := runtimeSetLockOwnerReadObserver + defer func() { runtimeSetLockOwnerReadObserver = priorObserver }() + ownerReadHeld := make(chan struct{}) + releaseOwnerRead := make(chan struct{}) + var firstOwnerRead atomic.Bool + runtimeSetLockOwnerReadObserver = func() { + if firstOwnerRead.CompareAndSwap(false, true) { + close(ownerReadHeld) + <-releaseOwnerRead + } + } + + waiterResult := make(chan bool, 1) + contenderToken := strings.Repeat("b", runtimeSetLockTokenSize*2) + go func() { + waiterResult <- runtimeSetTryReclaimLock(lock.path, contenderToken) + }() + <-ownerReadHeld + releaseErr := releaseRuntimeSetLock(lock) + close(releaseOwnerRead) + reclaimed := <-waiterResult + if releaseErr != nil { + t.Fatalf("release with live owner reader: %v", releaseErr) + } + if reclaimed { + t.Fatal("waiter reclaimed a proven-live owner") + } +} diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index 6d927fefa6..bab4a55b5e 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -390,17 +390,22 @@ def test_release_rejects_object_substitution_before_rename(self): def substitute_object_after_owner_close(owner_fd): nonlocal substituted - close(owner_fd) if ( owner_fd != lock[2] or substituted or not lock[0].exists() ): + close(owner_fd) return - substituted = True - owner_record = lock[0].read_text(encoding="utf-8") - lock[0].unlink() - lock[0].write_text(owner_record, encoding="utf-8") + owner_record = lock[0].read_bytes() + replacement = directory / "replacement-lock" + replacement.write_bytes(owner_record) + try: + close(owner_fd) + os.replace(replacement, lock[0]) + substituted = True + finally: + replacement.unlink(missing_ok=True) with mock.patch.object( _cli.os, From 079c6dcc7acab04b35b321632ead46d0830a65b8 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 22:01:49 +0200 Subject: [PATCH 07/13] fix(packages): make candidate probe test path portable Signed-off-by: Martin Vogel --- pkg/pypi/tests/test_cli.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index bab4a55b5e..5a3c819bc8 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -11,7 +11,7 @@ import time import unittest import zipfile -from pathlib import Path +from pathlib import Path, PureWindowsPath from unittest import mock @@ -208,15 +208,22 @@ def test_posix_probe_behavior_is_unchanged(self): class RuntimeSetTests(unittest.TestCase): def test_candidate_probe_authenticates_adjacent_runtime_assets(self): - with mock.patch.object(_cli.subprocess, "run") as run: - run.return_value.returncode = 0 - _cli._verify_candidate(Path("/tmp/codebase-memory-mcp")) - - self.assertEqual( - run.call_args.args[0], - ["/tmp/codebase-memory-mcp", "--verify-runtime-assets"], + candidates = ( + Path("/tmp/codebase-memory-mcp"), + PureWindowsPath("/tmp/codebase-memory-mcp"), ) + for candidate in candidates: + with self.subTest(candidate=candidate): + with mock.patch.object(_cli.subprocess, "run") as run: + run.return_value.returncode = 0 + _cli._verify_candidate(candidate) + + self.assertEqual( + run.call_args.args[0], + [str(candidate), "--verify-runtime-assets"], + ) + def test_runtime_readiness_rejects_multiply_linked_cache_leaves(self): with tempfile.TemporaryDirectory() as root, mock.patch.object( _cli, "_cache_dir", return_value=Path(root) From df2088a7abd6c22d188112b81573fbc3e65fbc94 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 22:26:53 +0200 Subject: [PATCH 08/13] fix(ci): pin bash for Windows fixture subprocesses Signed-off-by: Martin Vogel --- tests/test_smoke_fixture_contract.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/test_smoke_fixture_contract.sh b/tests/test_smoke_fixture_contract.sh index 177d386211..f793b299bf 100755 --- a/tests/test_smoke_fixture_contract.sh +++ b/tests/test_smoke_fixture_contract.sh @@ -5,7 +5,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" -python3 - "$ROOT" <<'PY' +python3 - "$ROOT" "$BASH" <<'PY' from __future__ import annotations import hashlib @@ -21,6 +21,7 @@ import urllib.request root = pathlib.Path(sys.argv[1]) +bash_executable = pathlib.Path(sys.argv[2]) failures: list[str] = [] @@ -58,7 +59,10 @@ windows_installer = read("install.ps1") # Keep the security-boundary archive checks runnable as a focused, network-free # contract while still making them part of the canonical smoke-fixture gate. release_archive_contract = subprocess.run( - ["bash", str(root / "tests/test_release_archive_extractor_contract.sh")], + [ + str(bash_executable), + str(root / "tests/test_release_archive_extractor_contract.sh"), + ], cwd=root, text=True, stdout=subprocess.PIPE, @@ -844,8 +848,16 @@ with tempfile.TemporaryDirectory(prefix="cbm-release-extract-") as extract_temp: package_environment = os.environ.copy() package_environment["BUILD_DIR"] = str(package_build) packaged = subprocess.run( - ["bash", str(root / "scripts/package-release.sh"), "linux", "amd64", - "--variant", "ui", "--out-dir", str(package_output)], + [ + str(bash_executable), + str(root / "scripts/package-release.sh"), + "linux", + "amd64", + "--variant", + "ui", + "--out-dir", + str(package_output), + ], cwd=root, env=package_environment, text=True, From bc4da946966cf76259f85aa3b257bbd8fcffb484 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 23:03:20 +0200 Subject: [PATCH 09/13] fix(ci): eliminate bare bash lookup in contracts Signed-off-by: Martin Vogel --- tests/test_package_runtime_assets_contract.sh | 5 ++-- ...test_release_archive_extractor_contract.sh | 11 ++++++-- tests/test_venue_parity_contract.sh | 25 +++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/tests/test_package_runtime_assets_contract.sh b/tests/test_package_runtime_assets_contract.sh index 04446f82ca..8ef4a36c83 100644 --- a/tests/test_package_runtime_assets_contract.sh +++ b/tests/test_package_runtime_assets_contract.sh @@ -7,7 +7,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -python3 - "$ROOT" <<'PY' +python3 - "$ROOT" "$BASH" <<'PY' from __future__ import annotations import json @@ -26,6 +26,7 @@ import zipfile root = pathlib.Path(sys.argv[1]) +bash_executable = pathlib.Path(sys.argv[2]) failures: list[str] = [] @@ -382,7 +383,7 @@ def package_fixture( archive_name = f"codebase-memory-mcp{'-ui' if variant == 'ui' else ''}-{host_goos}-fixture{archive_suffix}" packaged = subprocess.run( [ - "bash", + str(bash_executable), str(root / "scripts/package-release.sh"), host_goos, "fixture", diff --git a/tests/test_release_archive_extractor_contract.sh b/tests/test_release_archive_extractor_contract.sh index 60ce4ed270..ddb184d7b7 100644 --- a/tests/test_release_archive_extractor_contract.sh +++ b/tests/test_release_archive_extractor_contract.sh @@ -6,7 +6,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" -python3 - "$ROOT" <<'PY' +python3 - "$ROOT" "$BASH" <<'PY' from __future__ import annotations import csv @@ -25,6 +25,7 @@ from typing import Dict, List, Optional, Tuple root = pathlib.Path(sys.argv[1]) +bash_executable = pathlib.Path(sys.argv[2]) extractor = root / "scripts/ci/extract-release-archives.sh" unix_targets = ( "linux-amd64", @@ -307,7 +308,13 @@ def invoke( args: Optional[List[str]] = None, ) -> subprocess.CompletedProcess: return subprocess.run( - ["bash", str(extractor), str(archive_dir), str(output_dir), *(args or count_args)], + [ + str(bash_executable), + str(extractor), + str(archive_dir), + str(output_dir), + *(args or count_args), + ], cwd=root, text=True, stdout=subprocess.PIPE, diff --git a/tests/test_venue_parity_contract.sh b/tests/test_venue_parity_contract.sh index 48c4ea2ef6..d137c01a54 100755 --- a/tests/test_venue_parity_contract.sh +++ b/tests/test_venue_parity_contract.sh @@ -318,6 +318,31 @@ for local, pattern, why in LOCAL_REQUIRED: if path.exists() and not re.search(pattern, path.read_text()): failures.append(f"{local}: missing required `{pattern}` — {why}") +# Native Windows process lookup searches system locations before PATH. A bare +# `bash` argv launched by embedded Python can therefore select the WSL alias +# instead of the MSYS2 Bash that is already running this contract. Every +# shell-hosted Python block must receive the current `$BASH` as an explicit +# argument and launch that resolved path. Enforce this textually on every +# maintained shell surface so the rule is deterministic on non-Windows hosts +# too, rather than relying on a runner to happen to have the WSL alias. +BARE_BASH_ARGV = re.compile(r"""[\[(]\s*["']bash["']\s*,""") +shell_surfaces = list(root.glob("*.sh")) +for directory in ("tests", "scripts", "test-infrastructure", "pkg"): + base = root / directory + if not base.exists(): + continue + shell_surfaces.extend(base.rglob("*.sh")) +for path in sorted(shell_surfaces): + text = path.read_text(encoding="utf-8") + for match in BARE_BASH_ARGV.finditer(text): + number = text.count("\n", 0, match.start()) + 1 + relative = path.relative_to(root).as_posix() + failures.append( + f"{relative}:{number}: embedded Python must launch the current " + "$BASH by explicit argv path; bare `bash` is WSL-alias-sensitive " + "under native Windows process lookup" + ) + if failures: print("VENUE PARITY CONTRACT VIOLATED — one harness, every venue:") for failure in failures: From 0ce2c84eb2c02c9fe478b0c8b6345e6a1e610176 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 23:34:24 +0200 Subject: [PATCH 10/13] fix(ci): create real UI symlink fixture on Windows Signed-off-by: Martin Vogel --- tests/test_ui_asset_pack_contract.sh | 38 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/test_ui_asset_pack_contract.sh b/tests/test_ui_asset_pack_contract.sh index 6927027012..0fdeb3153b 100644 --- a/tests/test_ui_asset_pack_contract.sh +++ b/tests/test_ui_asset_pack_contract.sh @@ -8,6 +8,22 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" WORK="$(mktemp -d "${TMPDIR:-/tmp}/cbm-ui-pack-test.XXXXXX")" trap 'rm -rf "$WORK"' EXIT +make_real_dir_link() { + local target="$1" link="$2" + if MSYS=winsymlinks:nativestrict ln -s -- "$target" "$link" 2>/dev/null && + [ -L "$link" ]; then + return 0 + fi + rm -rf -- "$link" + case "$(uname -s)" in + MINGW* | MSYS* | CYGWIN*) + cmd //c "mklink /J \"$(cygpath -w "$link")\" \"$(cygpath -w "$target")\"" \ + >/dev/null 2>&1 || true + ;; + esac + [ -L "$link" ] +} + mkdir -p "$WORK/a/assets" "$WORK/b/assets" printf '%s\n' 'Codebase Memory — Graph' > "$WORK/a/index.html" printf '%s\n' 'console.log("transparent-ui-pack")' > "$WORK/a/assets/app.js" @@ -122,12 +138,26 @@ assert pack_path.name == f"cbm-ui-{digest}.pack" PY # A source symlink makes the build depend on bytes outside the declared dist -# tree. It is rejected rather than followed or silently omitted. -ln -s ../index.html "$WORK/a/assets/linked.html" -if node "$ROOT/scripts/pack-ui-assets.mjs" \ - "$WORK/a" "$WORK/a-out" "$WORK/rejected.c" 2>/dev/null; then +# tree. MSYS2's default `ln -s` silently deep-copies its target, so require a +# real native link and fall back to an NTFS junction before testing rejection. +mkdir "$WORK/outside-assets" +printf '%s\n' 'outside the declared UI root' > "$WORK/outside-assets/linked.html" +if ! make_real_dir_link "$WORK/outside-assets" "$WORK/a/assets/linked"; then + echo "FAIL: could not create a real symlink/junction fixture" >&2 + exit 1 +fi +if rejected_output="$(node "$ROOT/scripts/pack-ui-assets.mjs" \ + "$WORK/a" "$WORK/a-out" "$WORK/rejected.c" 2>&1)"; then echo "FAIL: UI pack generator accepted a symlink" >&2 exit 1 fi +case "$rejected_output" in +*"symbolic links are not allowed"*) ;; +*) + echo "FAIL: UI pack generator rejected the symlink for the wrong reason" >&2 + printf '%s\n' "$rejected_output" >&2 + exit 1 + ;; +esac echo "PASS: deterministic, transparent, hash-bound UI asset pack" From 9099bc383fc4386b751483d7866bf71c7380ef3d Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sat, 8 Aug 2026 23:58:58 +0200 Subject: [PATCH 11/13] fix(ci): install zip for Windows package contract Signed-off-by: Martin Vogel --- .github/workflows/_test.yml | 1 + tests/test_package_runtime_assets_contract.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index c6d98127b6..3b95629753 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -425,6 +425,7 @@ jobs: mingw-w64-clang-${{ matrix.pkg }}-ccache make git + zip - name: Create protected per-user temp root # One implementation, shared with _soak.yml and the VM leg diff --git a/tests/test_package_runtime_assets_contract.sh b/tests/test_package_runtime_assets_contract.sh index 8ef4a36c83..af910818fd 100644 --- a/tests/test_package_runtime_assets_contract.sh +++ b/tests/test_package_runtime_assets_contract.sh @@ -40,6 +40,7 @@ def require(condition: bool, message: str) -> None: package_release = read("scripts/package-release.sh") +test_workflow = read(".github/workflows/_test.yml") strip_position = package_release.find('strip_release_binary "$STAGED_BINARY"') probe_position = package_release.find('"$STAGED_BINARY" --verify-runtime-assets') tar_position = package_release.find('tar -czf "$OUT_DIR/$NAME.tar.gz"') @@ -67,6 +68,15 @@ require( and 'zip -q "$OUT_DIR/$NAME.zip" "${ARCHIVE_MEMBERS[@]}"' in package_release, "Windows release archive must be created from the same private directory that was probed", ) +windows_test_job = re.search( + r"(?ms)^ test-windows:\n(?P.*?)(?=^ [a-zA-Z0-9_-]+:\n)", + test_workflow, +) +require( + windows_test_job is not None + and re.search(r"(?m)^\s+zip\s*$", windows_test_job.group("body")) is not None, + "Windows test jobs that exercise package-release.sh must install zip", +) homebrew = read("pkg/homebrew/Formula/codebase-memory-mcp.rb") From b0279a010c1bdd00192787651f0df0fd8c3f913c Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 9 Aug 2026 00:31:14 +0200 Subject: [PATCH 12/13] fix(ci): resolve VT curl through current Bash Signed-off-by: Martin Vogel --- scripts/ci/check-virustotal.sh | 8 +++++++- tests/test_vt_gate_zero_tolerance_contract.sh | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/ci/check-virustotal.sh b/scripts/ci/check-virustotal.sh index 8f446ad30d..378594719a 100755 --- a/scripts/ci/check-virustotal.sh +++ b/scripts/ci/check-virustotal.sh @@ -17,7 +17,7 @@ command -v curl >/dev/null 2>&1 || { exit 1 } -python3 - <<'PY' +python3 - "$BASH" <<'PY' from __future__ import annotations import csv @@ -37,6 +37,9 @@ from dataclasses import dataclass from typing import Deque, Dict, List, Optional, Sequence, Tuple +bash_executable = pathlib.Path(sys.argv[1]) + + class GateError(Exception): pass @@ -533,6 +536,9 @@ def fetch_analysis(api_key: str, analysis_id: str, curl_timeout: int) -> Optiona encoded_id = urllib.parse.quote(analysis_id, safe="") response = subprocess.run( [ + str(bash_executable), + "-c", + 'exec curl "$@"', "curl", "-sf", "--max-time", diff --git a/tests/test_vt_gate_zero_tolerance_contract.sh b/tests/test_vt_gate_zero_tolerance_contract.sh index f7c29a3407..e89d837c02 100644 --- a/tests/test_vt_gate_zero_tolerance_contract.sh +++ b/tests/test_vt_gate_zero_tolerance_contract.sh @@ -20,6 +20,14 @@ for needle in defender-endpoint-verification av-endpoint-verify false-positive f fi done +# Source-contract patterns intentionally retain shell variables literally. +# shellcheck disable=SC2016 +grep -Fq 'python3 - "$BASH"' "$GATE" || + fail "gate must pass its current Bash explicitly to native Windows Python" +# shellcheck disable=SC2016 +grep -Fq 'exec curl "$@"' "$GATE" || + fail "gate must resolve curl through its current Bash without interpolating argv" + mkdir -p "$FIX/bin" "$FIX/work/binaries/objects" "$FIX/responses" printf 'release-bytes\n' > "$FIX/work/binaries/objects/probe" printf 'second-object\n' > "$FIX/work/binaries/objects/probe2" From 5c18e3f2d04e4a8870d4896c3bd0978a61411db6 Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Sun, 9 Aug 2026 01:11:29 +0200 Subject: [PATCH 13/13] ci: scan only UI artifacts in dry runs Signed-off-by: Martin Vogel --- .github/workflows/_build.yml | 42 +++++++---- .github/workflows/_smoke.yml | 26 ++++--- .github/workflows/dry-run.yml | 9 ++- scripts/ci/extract-release-archives.sh | 42 +++++++---- ...test_release_archive_extractor_contract.sh | 72 +++++++++++++++++-- tests/test_windows_bundle_contract.sh | 7 +- 6 files changed, 151 insertions(+), 47 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index e6bad802da..3f55dc2678 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -1,4 +1,4 @@ -# Reusable: build binaries (standard + UI) on all platforms +# Reusable: build release binaries on all supported platforms name: Build on: @@ -12,6 +12,10 @@ on: description: 'Generate build provenance attestations for release artifacts' type: boolean default: false + ui_only: + description: 'Build only the UI runtime archives' + type: boolean + default: false permissions: contents: read @@ -64,6 +68,7 @@ jobs: node-version: "22" - name: Build standard binary + if: ${{ !inputs.ui_only }} env: VERSION: ${{ inputs.version }} CC: ${{ matrix.cc }} @@ -75,17 +80,16 @@ jobs: scripts/build.sh "CC=$CC" "CXX=$CXX" fi - - name: Ad-hoc sign macOS binary - if: startsWith(matrix.os, 'macos') + - name: Ad-hoc sign macOS standard binary + if: ${{ !inputs.ui_only && startsWith(matrix.os, 'macos') }} run: codesign --sign - --force build/c/codebase-memory-mcp - # Archive layout/name live in the ONE canonical script the local - # artifact-flow smoke lane also runs (venue-parity contract). - name: Archive standard binary (canonical package-release.sh) + if: ${{ !inputs.ui_only }} run: scripts/package-release.sh ${{ matrix.goos }} ${{ matrix.goarch }} - name: Attest standard binary provenance - if: ${{ inputs.attest }} + if: ${{ inputs.attest && !inputs.ui_only }} uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: codebase-memory-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz @@ -110,6 +114,8 @@ jobs: if: matrix.goos == 'linux' && matrix.goarch == 'amd64' run: scripts/security-ui.sh + # Archive layout/name live in the ONE canonical script the local + # artifact-flow smoke lane also runs (venue-parity contract). - name: Archive UI binary (canonical package-release.sh) run: scripts/package-release.sh ${{ matrix.goos }} ${{ matrix.goarch }} --variant ui @@ -145,6 +151,7 @@ jobs: node-version: "22" - name: Build standard binary + if: ${{ !inputs.ui_only }} shell: msys2 {0} env: VERSION: ${{ inputs.version }} @@ -155,14 +162,13 @@ jobs: scripts/build.sh CC=clang CXX=clang++ fi - # Archive layout/name live in the ONE canonical script the local - # artifact-flow smoke lane also runs (venue-parity contract). - name: Archive standard binary (canonical package-release.sh) + if: ${{ !inputs.ui_only }} shell: msys2 {0} run: scripts/package-release.sh windows amd64 CC=clang CXX=clang++ - name: Attest standard binary provenance - if: ${{ inputs.attest }} + if: ${{ inputs.attest && !inputs.ui_only }} uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: codebase-memory-mcp-windows-amd64.zip @@ -178,6 +184,8 @@ jobs: scripts/build.sh --with-ui CC=clang CXX=clang++ fi + # Archive layout/name live in the ONE canonical script the local + # artifact-flow smoke lane also runs (venue-parity contract). - name: Archive UI binary (canonical package-release.sh) shell: msys2 {0} run: scripts/package-release.sh windows amd64 --variant ui CC=clang CXX=clang++ @@ -218,6 +226,7 @@ jobs: node-version: "22" - name: Build standard binary + if: ${{ !inputs.ui_only }} shell: msys2 {0} env: VERSION: ${{ inputs.version }} @@ -229,11 +238,12 @@ jobs: fi - name: Archive standard binary (canonical package-release.sh) + if: ${{ !inputs.ui_only }} shell: msys2 {0} run: scripts/package-release.sh windows arm64 CC=clang CXX=clang++ - name: Attest standard binary provenance - if: ${{ inputs.attest }} + if: ${{ inputs.attest && !inputs.ui_only }} uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: codebase-memory-mcp-windows-arm64.zip @@ -288,6 +298,7 @@ jobs: node-version: "22" - name: Build standard binary (static) + if: ${{ !inputs.ui_only }} env: VERSION: ${{ inputs.version }} run: | @@ -297,16 +308,18 @@ jobs: scripts/build.sh CC=gcc CXX=g++ STATIC=1 fi - - name: Verify static linking + - name: Verify standard static linking + if: ${{ !inputs.ui_only }} run: | file build/c/codebase-memory-mcp ldd build/c/codebase-memory-mcp 2>&1 | grep -q "not a dynamic executable" || ldd build/c/codebase-memory-mcp 2>&1 | grep -q "statically linked" - name: Archive standard binary (canonical package-release.sh) + if: ${{ !inputs.ui_only }} run: scripts/package-release.sh linux ${{ matrix.arch }}-portable - name: Attest standard binary provenance - if: ${{ inputs.attest }} + if: ${{ inputs.attest && !inputs.ui_only }} uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 with: subject-path: codebase-memory-mcp-linux-${{ matrix.arch }}-portable.tar.gz @@ -321,6 +334,11 @@ jobs: scripts/build.sh --with-ui CC=gcc CXX=g++ STATIC=1 fi + - name: Verify UI static linking + run: | + file build/c/codebase-memory-mcp + ldd build/c/codebase-memory-mcp 2>&1 | grep -q "not a dynamic executable" || ldd build/c/codebase-memory-mcp 2>&1 | grep -q "statically linked" + - name: Archive UI binary (canonical package-release.sh) run: scripts/package-release.sh linux ${{ matrix.arch }}-portable --variant ui diff --git a/.github/workflows/_smoke.yml b/.github/workflows/_smoke.yml index d979697044..fca2f04012 100644 --- a/.github/workflows/_smoke.yml +++ b/.github/workflows/_smoke.yml @@ -1,4 +1,4 @@ -# Reusable: smoke test every binary (standard + UI, all platforms) +# Reusable: smoke test shipped release binaries on all platforms name: Smoke on: @@ -8,6 +8,10 @@ on: description: 'Smoke the shipped binaries on the broad platform matrix (extra OS versions) instead of the core set' type: boolean default: false + ui_only: + description: 'Smoke only the UI runtime archives' + type: boolean + default: false permissions: contents: read @@ -30,6 +34,7 @@ jobs: id: set env: BROAD: ${{ inputs.broad_platforms }} + UI_ONLY: ${{ inputs.ui_only }} run: | CORE_UNIX='[ {"os":"ubuntu-latest","goos":"linux","goarch":"amd64"}, @@ -72,13 +77,14 @@ jobs: WIN=$(jq -cn --argjson a "$CORE_WIN" '$a') PORTABLE=$(jq -cn --argjson a "$CORE_PORTABLE" '$a') fi - # Expand each OS entry over both variants into a FLAT include list so - # every {os,variant} runs as its own job. A {variant:[...],include:$LIST} - # matrix collapses under GitHub's include-merge (later os entries - # overwrite the variant combos last-wins), which silently dropped every - # OS except the last — e.g. only windows-11-arm ran, never windows-latest. - # Building the cartesian explicitly avoids that. - VARIANTS='["standard","ui"]' + # Keep the variant dimension explicit because the smoke wrappers use it + # to require and validate the UI pack. VT-focused dry runs select only + # UI; release compatibility retains both variants until migration. + if [ "$UI_ONLY" = "true" ]; then + VARIANTS='["ui"]' + else + VARIANTS='["standard","ui"]' + fi UNIX_M=$(jq -cn --argjson a "$UNIX" --argjson v "$VARIANTS" '{include:[$a[] as $o | $v[] as $t | ($o + {variant:$t})]}') WIN_M=$(jq -cn --argjson a "$WIN" --argjson v "$VARIANTS" '{include:[$a[] as $o | $v[] as $t | ($o + {variant:$t})]}') PORTABLE_M=$(jq -cn --argjson a "$PORTABLE" --argjson v "$VARIANTS" '{include:[$a[] as $o | $v[] as $t | ($o + {variant:$t})]}') @@ -133,8 +139,8 @@ jobs: scripts/security-install.sh "$BIN" scripts/security-network.sh "$BIN" - - name: MCP robustness test (linux-amd64 standard only) - if: matrix.variant == 'standard' && matrix.goos == 'linux' && matrix.goarch == 'amd64' + - name: MCP robustness test (selected linux-amd64 runtime) + if: matrix.goos == 'linux' && matrix.goarch == 'amd64' && (inputs.ui_only || matrix.variant == 'standard') run: | BIN="$RUNNER_TEMP/cbm-artifact/codebase-memory-mcp" scripts/security-fuzz.sh "$BIN" diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 19d6f45152..e9134c6d3e 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -91,6 +91,7 @@ jobs: uses: ./.github/workflows/_build.yml with: attest: false + ui_only: true # ── Smoke test every binary ──────────────────────────────────── # Run unless builds were skipped or a build leg FAILED. Every build leg @@ -102,6 +103,7 @@ jobs: uses: ./.github/workflows/_smoke.yml with: broad_platforms: true + ui_only: true # ── Soak tests (optional, parallel with smoke) ──────────────── soak: @@ -151,10 +153,11 @@ jobs: - name: Extract archive artifacts run: | scripts/ci/extract-release-archives.sh assets binaries \ - --expect-archives=16 \ - --expect-binaries=16 \ + --archive-scope=ui \ + --expect-archives=8 \ + --expect-binaries=8 \ --expect-packs=8 \ - --expect-runtime-files=64 + --expect-runtime-files=32 - name: VirusTotal scan uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0 diff --git a/scripts/ci/extract-release-archives.sh b/scripts/ci/extract-release-archives.sh index 0dbd8091ca..487857277b 100755 --- a/scripts/ci/extract-release-archives.sh +++ b/scripts/ci/extract-release-archives.sh @@ -4,7 +4,7 @@ # # Usage: # extract-release-archives.sh \ -# [--expect-archives=N] [--expect-binaries=N] [--expect-packs=N] \ +# [--archive-scope=all|ui] [--expect-archives=N] [--expect-binaries=N] [--expect-packs=N] \ # [--expect-runtime-files=N] # # The output directory is published as one atomic bundle: @@ -18,7 +18,7 @@ set -euo pipefail if [ "$#" -lt 2 ]; then - echo "Usage: $0 [--expect-archives=N] [--expect-binaries=N] [--expect-packs=N] [--expect-runtime-files=N]" >&2 + echo "Usage: $0 [--archive-scope=all|ui] [--expect-archives=N] [--expect-binaries=N] [--expect-packs=N] [--expect-runtime-files=N]" >&2 exit 2 fi @@ -70,18 +70,21 @@ UNIX_TARGETS = ( "linux-arm64-portable", ) WINDOWS_TARGETS = ("windows-amd64", "windows-arm64") -CANONICAL_ARCHIVES = frozenset( +UI_ARCHIVES = frozenset( [ - f"codebase-memory-mcp{suffix}-{target}.tar.gz" - for suffix in ("", "-ui") + f"codebase-memory-mcp-ui-{target}.tar.gz" for target in UNIX_TARGETS ] + [ - f"codebase-memory-mcp{suffix}-{target}.zip" - for suffix in ("", "-ui") + f"codebase-memory-mcp-ui-{target}.zip" for target in WINDOWS_TARGETS ] ) +STANDARD_ARCHIVES = frozenset( + [f"codebase-memory-mcp-{target}.tar.gz" for target in UNIX_TARGETS] + + [f"codebase-memory-mcp-{target}.zip" for target in WINDOWS_TARGETS] +) +CANONICAL_ARCHIVES = UI_ARCHIVES | STANDARD_ARCHIVES PACK_NAME = re.compile(r"cbm-ui-([0-9a-f]{64})\.pack\Z") SAFE_LABEL = re.compile(r"[^A-Za-z0-9._-]+") SAFE_ASSET_PATH = re.compile(rb"\A[A-Za-z0-9._/-]+\Z") @@ -290,13 +293,22 @@ class ObjectStore: def parse_arguments( argv: Sequence[str], -) -> Tuple[pathlib.Path, pathlib.Path, Dict[str, Optional[int]]]: +) -> Tuple[pathlib.Path, pathlib.Path, Dict[str, Optional[int]], str]: archive_dir = pathlib.Path(argv[1]).absolute() output_dir = pathlib.Path(argv[2]).absolute() expected: Dict[str, Optional[int]] = {value: None for value in COUNT_OPTIONS.values()} seen: set[str] = set() + archive_scope = "all" for argument in argv[3:]: option, separator, raw_value = argument.partition("=") + if separator and option == "--archive-scope": + if "archive_scope" in seen: + raise ContractError(f"duplicate option: {option}") + if raw_value not in {"all", "ui"}: + raise ContractError("--archive-scope must be all or ui") + archive_scope = raw_value + seen.add("archive_scope") + continue if not separator or option not in COUNT_OPTIONS: raise ContractError(f"unknown option: {argument}") key = COUNT_OPTIONS[option] @@ -306,7 +318,7 @@ def parse_arguments( raise ContractError(f"{option} requires a non-negative integer") expected[key] = int(raw_value) seen.add(key) - return archive_dir, output_dir, expected + return archive_dir, output_dir, expected, archive_scope def validate_namespace(archive_name: str, names: Iterable[str]) -> Tuple[str, Dict[str, str]]: @@ -674,7 +686,7 @@ def write_tsv( def main(argv: Sequence[str]) -> None: - archive_dir, output_dir, expected = parse_arguments(argv) + archive_dir, output_dir, expected, archive_scope = parse_arguments(argv) if not archive_dir.is_dir() or archive_dir.is_symlink(): raise ContractError(f"archive directory is not a regular directory: {archive_dir}") if output_dir.name in ("", ".", ".."): @@ -687,11 +699,13 @@ def main(argv: Sequence[str]) -> None: archive_paths = sorted(archive_dir.iterdir(), key=lambda candidate: candidate.name) actual_names = {path.name for path in archive_paths} - if actual_names != CANONICAL_ARCHIVES or len(archive_paths) != len(CANONICAL_ARCHIVES): - missing = sorted(CANONICAL_ARCHIVES - actual_names) - unexpected = sorted(actual_names - CANONICAL_ARCHIVES) + expected_names = UI_ARCHIVES if archive_scope == "ui" else CANONICAL_ARCHIVES + if actual_names != expected_names or len(archive_paths) != len(expected_names): + missing = sorted(expected_names - actual_names) + unexpected = sorted(actual_names - expected_names) raise ContractError( - f"archive namespace mismatch: expected exact canonical 16; missing={missing}, unexpected={unexpected}" + f"archive namespace mismatch: expected exact canonical {len(expected_names)} " + f"({archive_scope}); missing={missing}, unexpected={unexpected}" ) for path in archive_paths: mode = path.lstat().st_mode diff --git a/tests/test_release_archive_extractor_contract.sh b/tests/test_release_archive_extractor_contract.sh index ddb184d7b7..67872475ef 100644 --- a/tests/test_release_archive_extractor_contract.sh +++ b/tests/test_release_archive_extractor_contract.sh @@ -50,6 +50,7 @@ archive_names = tuple( ] ) ) +ui_archive_names = tuple(name for name in archive_names if name.startswith("codebase-memory-mcp-ui-")) expected_base_counts = { "archives": 16, "binaries": 16, @@ -60,6 +61,19 @@ count_args = [ f"--expect-{key.replace('_', '-')}={value}" for key, value in expected_base_counts.items() ] +ui_expected_base_counts = { + "archives": 8, + "binaries": 8, + "packs": 8, + "runtime_files": 32, +} +ui_count_args = [ + "--archive-scope=ui", + *[ + f"--expect-{key.replace('_', '-')}={value}" + for key, value in ui_expected_base_counts.items() + ], +] asset_payloads = { "/assets/app.css": b"body{color:#123}\n", "/assets/app.js": b"globalThis.__CBM_UI__ = true;\n", @@ -233,6 +247,7 @@ Association = Tuple[str, str, str, str] def build_matrix( archive_dir: pathlib.Path, mutation: Optional[str] = None, + selected_names: Tuple[str, ...] = archive_names, ) -> Dict[Association, bytes]: archive_dir.mkdir(parents=True) expected: Dict[Association, bytes] = {} @@ -253,7 +268,7 @@ def build_matrix( "zip_duplicate": "codebase-memory-mcp-ui-windows-amd64.zip", } target = targets.get(mutation) - for name in archive_names: + for name in selected_names: if mutation == "missing_archive" and name == "codebase-memory-mcp-linux-amd64.tar.gz": continue pack_mutation = mutation if name == target and mutation and mutation.startswith("bad_pack_") else None @@ -350,8 +365,8 @@ def assert_clean_failure(case_root: pathlib.Path, mutation: str, message: str) - fail(f"{mutation}: rejected archives left a partial atomic scan bundle") -if len(archive_names) != 16: - fail("fixture matrix itself no longer describes 16 canonical archives") +if len(archive_names) != 16 or len(ui_archive_names) != 8: + fail("fixture matrix no longer describes 16 canonical archives / eight UI archives") with tempfile.TemporaryDirectory(prefix="cbm-release-scan-contract-") as temporary: work = pathlib.Path(temporary) @@ -428,6 +443,34 @@ with tempfile.TemporaryDirectory(prefix="cbm-release-scan-contract-") as tempora if row["sha256"] != hashlib.sha256(data).hexdigest() or int(row["size"]) != len(data): fail(f"scan-set digest/size does not bind {row['scan_path']}") + ui_only = work / "ui-only" + ui_expected = build_matrix(ui_only / "archives", selected_names=ui_archive_names) + ui_output = ui_only / "scan" + ui_result = invoke(ui_only / "archives", ui_output, ui_count_args) + if ui_result.returncode != 0: + fail(f"canonical UI-only matrix was rejected: {ui_result.stdout.strip()}") + ui_metadata, ui_rows = parse_manifest( + ui_output / "associations.tsv", "cbm-release-scan-associations-v2" + ) + ui_associations = 8 + 48 + expected_pack_assets + if ui_metadata != { + **ui_expected_base_counts, + "pack_assets": expected_pack_assets, + "associations": ui_associations, + "scan_objects": len(set(ui_expected.values())), + }: + fail(f"UI-only association metadata mismatch: {ui_metadata!r}") + if len(ui_rows) != ui_associations or {row["variant"] for row in ui_rows} != {"ui"}: + fail("UI-only scan scope does not cover exactly the eight UI archives") + ui_as_all_output = ui_only / "scan-as-all" + ui_as_all = invoke(ui_only / "archives", ui_as_all_output, count_args) + if ( + ui_as_all.returncode == 0 + or "expected exact canonical 16 (all)" not in ui_as_all.stdout + or ui_as_all_output.exists() + ): + fail("UI-only archives must require an explicit closed archive scope") + failure_cases = { "bad_hash": "UI pack digest does not match its filename", "bad_pack_magic": "CBMUIPK header/offset contract failed", @@ -436,13 +479,13 @@ with tempfile.TemporaryDirectory(prefix="cbm-release-scan-contract-") as tempora "bad_pack_mime": "CBMUIPK MIME/path contract failed", "directory": "non-regular archive member", "extra_pack": "exactly one UI pack", - "missing_archive": "expected exact canonical 16", + "missing_archive": "expected exact canonical 16 (all)", "missing_pack": "exactly one UI pack", "nested_member": "unexpected archive member", "oversized_archive": "archive exceeds 536870912 byte ceiling", "tar_symlink": "non-regular archive member", "tar_duplicate": "duplicate archive member", - "unexpected_archive": "expected exact canonical 16", + "unexpected_archive": "expected exact canonical 16 (all)", "zip_directory": "non-regular archive member", "zip_duplicate": "duplicate archive member", "zip_symlink": "non-regular archive member", @@ -473,8 +516,23 @@ for ceiling in ("MAX_ARCHIVE_BYTES", "MAX_MEMBER_BYTES", "MAX_TOTAL_MEMBER_BYTES release = (root / ".github/workflows/release.yml").read_text(encoding="utf-8") dry_run = (root / ".github/workflows/dry-run.yml").read_text(encoding="utf-8") -for workflow_name, workflow in (("release", release), ("dry-run", dry_run)): - for argument in count_args: +build = (root / ".github/workflows/_build.yml").read_text(encoding="utf-8") +smoke = (root / ".github/workflows/_smoke.yml").read_text(encoding="utf-8") +if "ui_only:" not in build or build.count("if: ${{ !inputs.ui_only }}") < 4: + fail("canonical build workflow must gate every standard build family behind ui_only") +if build.count("--variant ui") != 4: + fail("canonical build workflow must package exactly its four UI build families") +if "UI_ONLY: ${{ inputs.ui_only }}" not in smoke or "VARIANTS='[\"ui\"]'" not in smoke: + fail("release smoke matrix must support an explicit UI-only selection") +if dry_run.count("ui_only: true") != 2 or "--archive-scope=ui" not in dry_run: + fail("dry-run must build, smoke and extract only the UI runtime set") +if "--archive-scope=ui" in release: + fail("release workflow must retain its dual-variant compatibility scope") +for workflow_name, workflow, arguments in ( + ("release", release, count_args), + ("dry-run", dry_run, ui_count_args), +): + for argument in arguments: if argument not in workflow: fail(f"{workflow_name} workflow does not assert {argument}") for required in ( diff --git a/tests/test_windows_bundle_contract.sh b/tests/test_windows_bundle_contract.sh index 5acb8b9d51..1661dfeda6 100644 --- a/tests/test_windows_bundle_contract.sh +++ b/tests/test_windows_bundle_contract.sh @@ -148,7 +148,8 @@ require( ) # Each archive variant is still produced through the ONE canonical packaging -# entry, so the five-or-six-file layout above governs all of them. +# entry, so the five-or-six-file layout above governs all of them. The standard +# calls are conditionally skipped by VT-focused UI-only dry runs. build_workflow = read(".github/workflows/_build.yml") for archive, call, ui in ( ("codebase-memory-mcp-windows-amd64.zip", "scripts/package-release.sh windows amd64", False), @@ -164,6 +165,10 @@ for archive, call, ui in ( re.search(pattern, build_workflow) is not None, f"_build.yml must produce {archive} via the canonical packaging entry ('{call}')", ) +require( + build_workflow.count("if: ${{ !inputs.ui_only }}") >= 4, + "_build.yml must skip every standard Windows build/archive in UI-only mode", +) # ── 3. install.ps1 delegates every runtime mutation to the candidate ───────── installer = read("install.ps1")