Skip to content

Commit 4ed8d38

Browse files
authored
Merge pull request DeusData#1508 from DeusData/revert/recentralize-runtime-assets
Re-embed runtime assets, ship one archive set, and tolerate a single Microsoft !ml verdict
2 parents 10cb0e0 + f36ab45 commit 4ed8d38

78 files changed

Lines changed: 3113 additions & 11652 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_build.yml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,13 +85,13 @@ jobs:
8585
# Archive layout/name live in the ONE canonical script the local
8686
# artifact-flow smoke lane also runs (venue-parity contract).
8787
- name: Archive release binary (canonical package-release.sh)
88-
run: scripts/package-release.sh ${{ matrix.goos }} ${{ matrix.goarch }} --variant ui
88+
run: scripts/package-release.sh ${{ matrix.goos }} ${{ matrix.goarch }}
8989

9090
- name: Attest release binary provenance
9191
if: ${{ inputs.attest }}
9292
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
9393
with:
94-
subject-path: codebase-memory-mcp-ui-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz
94+
subject-path: codebase-memory-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz
9595

9696
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
9797
with:
@@ -133,13 +133,13 @@ jobs:
133133
# artifact-flow smoke lane also runs (venue-parity contract).
134134
- name: Archive release binary (canonical package-release.sh)
135135
shell: msys2 {0}
136-
run: scripts/package-release.sh windows amd64 --variant ui CC=clang CXX=clang++
136+
run: scripts/package-release.sh windows amd64 CC=clang CXX=clang++
137137

138138
- name: Attest release binary provenance
139139
if: ${{ inputs.attest }}
140140
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
141141
with:
142-
subject-path: codebase-memory-mcp-ui-windows-amd64.zip
142+
subject-path: codebase-memory-mcp-windows-amd64.zip
143143

144144
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
145145
with:
@@ -183,13 +183,13 @@ jobs:
183183
184184
- name: Archive release binary (canonical package-release.sh)
185185
shell: msys2 {0}
186-
run: scripts/package-release.sh windows arm64 --variant ui CC=clang CXX=clang++
186+
run: scripts/package-release.sh windows arm64 CC=clang CXX=clang++
187187

188188
- name: Attest release binary provenance
189189
if: ${{ inputs.attest }}
190190
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
191191
with:
192-
subject-path: codebase-memory-mcp-ui-windows-arm64.zip
192+
subject-path: codebase-memory-mcp-windows-arm64.zip
193193

194194
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
195195
with:
@@ -235,13 +235,13 @@ jobs:
235235
ldd build/c/codebase-memory-mcp 2>&1 | grep -q "not a dynamic executable" || ldd build/c/codebase-memory-mcp 2>&1 | grep -q "statically linked"
236236
237237
- name: Archive release binary (canonical package-release.sh)
238-
run: scripts/package-release.sh linux ${{ matrix.arch }}-portable --variant ui
238+
run: scripts/package-release.sh linux ${{ matrix.arch }}-portable
239239

240240
- name: Attest release binary provenance
241241
if: ${{ inputs.attest }}
242242
uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
243243
with:
244-
subject-path: codebase-memory-mcp-ui-linux-${{ matrix.arch }}-portable.tar.gz
244+
subject-path: codebase-memory-mcp-linux-${{ matrix.arch }}-portable.tar.gz
245245

246246
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
247247
with:

‎.github/workflows/dry-run.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,9 @@ jobs:
151151
- name: Extract archive artifacts
152152
run: |
153153
scripts/ci/extract-release-archives.sh "$RUNNER_TEMP/release-archives" binaries \
154-
--archive-scope=ui \
155154
--expect-archives=8 \
156155
--expect-binaries=8 \
157-
--expect-packs=8 \
158-
--expect-runtime-files=32
156+
--expect-runtime-files=24
159157
160158
- name: VirusTotal scan
161159
uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0

‎.github/workflows/release.yml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -216,11 +216,9 @@ jobs:
216216
mkdir -p "$ARCHIVE_DIR" binaries
217217
gh release download "$VERSION" --dir "$ARCHIVE_DIR" --repo "$GITHUB_REPOSITORY" --pattern '*.tar.gz' --pattern '*.zip'
218218
scripts/ci/extract-release-archives.sh "$ARCHIVE_DIR" binaries \
219-
--archive-scope=ui \
220219
--expect-archives=8 \
221220
--expect-binaries=8 \
222-
--expect-packs=8 \
223-
--expect-runtime-files=32
221+
--expect-runtime-files=24
224222
225223
- name: Security audits on all unique extracted release objects
226224
run: |

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ private/
4848
build/
4949
node_modules/
5050
graph-ui/dist/
51+
# Generated by scripts/embed-frontend.sh into the source tree (the linker needs
52+
# it beside the other UI sources). Removed by scripts/clean.sh; never committed.
53+
src/ui/embedded_assets.c
5154

5255
# Generated reports
5356
BENCHMARK_REPORT.md

‎Makefile.cbm‎

Lines changed: 32 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -162,10 +162,18 @@ endif
162162
# link), this flag fixes the OUTCOME, and the composition gate proves it on the
163163
# shipped artifact. ELF-only: Apple's ld rejects -z noexecstack outright and it
164164
# is meaningless for PE, so it is gated rather than made "common".
165+
#
166+
# -z separate-code additionally keeps read-only DATA out of the executable
167+
# segment. GNU ld enables it by default on x86-64 but NOT on aarch64, so the
168+
# arm64 binaries emitted ONE R E PT_LOAD spanning the whole image: .rodata is
169+
# correctly marked A (not AX), but the kernel applies SEGMENT permissions, so
170+
# ~259 MB of tree-sitter parse tables was mapped executable at runtime while the
171+
# amd64 build of the same source mapped them R only. That is a large ROP gadget
172+
# surface for data that is never executed, and it is invisible to section flags.
165173
ELF_HARDENING_FLAGS :=
166174
ifeq ($(IS_LINUX),yes)
167175
ifneq ($(IS_MINGW),yes)
168-
ELF_HARDENING_FLAGS := -Wl,-z,noexecstack
176+
ELF_HARDENING_FLAGS := -Wl,-z,noexecstack -Wl,-z,separate-code
169177
endif
170178
endif
171179

@@ -358,19 +366,17 @@ GIT_SRCS = src/git/git_context.c
358366

359367
# CLI module (new)
360368
CLI_SRCS = src/cli/cli.c src/cli/progress_sink.c src/cli/hook_augment.c src/cli/client_adapter.c \
361-
src/cli/agent_clients.c src/cli/agent_profiles.c src/cli/integration_assets.c \
369+
src/cli/agent_clients.c src/cli/agent_profiles.c \
362370
src/cli/config_json_like.c src/cli/config_toml_edit.c src/cli/config_yaml_edit.c \
363371
src/cli/config_text_edit.c src/cli/activation_transaction.c
364372

365-
# UI module (graph visualization). Production-standard builds deliberately link
366-
# only the no-I/O pack stub. Test binaries substitute the real parser plus the
367-
# empty manifest so parser/install behavior remains covered without a UI pack.
373+
# UI module (graph visualization)
368374
UI_SRCS = \
369375
src/ui/config.c \
370376
src/ui/http_server.c \
371377
src/ui/layout3d.c \
372378
src/ui/httpd.c \
373-
src/ui/asset_pack_stub.c
379+
src/ui/embedded_stub.c
374380

375381
# mimalloc (vendored, global allocator override)
376382
#
@@ -458,7 +464,6 @@ YYJSON_SRC = vendored/yyjson/yyjson.c
458464

459465
# All production sources
460466
PROD_SRCS = $(FOUNDATION_SRCS) $(STORE_SRCS) $(CYPHER_SRCS) $(MCP_SRCS) $(DAEMON_SRCS) $(DISCOVER_SRCS) $(GRAPH_BUFFER_SRCS) $(PIPELINE_SRCS) $(SIMHASH_SRCS) $(SEMANTIC_SRCS) $(TRACES_SRCS) $(WATCHER_SRCS) $(GIT_SRCS) $(CLI_SRCS) $(UI_SRCS) $(YYJSON_SRC)
461-
TEST_PROD_SRCS = $(subst src/ui/asset_pack_stub.c,src/ui/asset_pack.c src/ui/asset_manifest_stub.c,$(PROD_SRCS))
462467
EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
463468
$(GRAMMAR_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC)
464469

@@ -476,17 +481,6 @@ EXISTING_C_SRCS = $(EXTRACTION_SRCS) $(LSP_SRCS) $(TS_RUNTIME_SRC) \
476481
PROJECT_HDRS = $(wildcard src/*.h src/*/*.h $(CBM_DIR)/*.h $(CBM_DIR)/lsp/*.h tests/*.h \
477482
tests/repro/*.h)
478483

479-
# ── Integration-asset integrity constant (generated, never hand-edited) ──
480-
# The binary embeds exactly ONE fact about the shipped integration templates:
481-
# the SHA-256 of assets/cbm-integrations.json, generated at build time (rule
482-
# below the BUILD_DIR definition; target lines expand immediately). Appended
483-
# to PROJECT_HDRS so every single-invocation target (test-runner,
484-
# test-repro-runner, test-runner-tsan, codebase-memory-mcp) rebuilds when the
485-
# asset changes — a stale constant would make every install fail verification.
486-
INTEGRATIONS_ASSET = assets/cbm-integrations.json
487-
INTEGRATIONS_HASH_HDR = $(BUILD_DIR)/generated/cbm_integrations_hash.h
488-
PROJECT_HDRS += $(INTEGRATIONS_HASH_HDR)
489-
490484
# ── Test sources ─────────────────────────────────────────────────
491485

492486
TEST_FOUNDATION_SRCS = \
@@ -702,11 +696,6 @@ ALL_TEST_SRCS =$(TEST_FOUNDATION_SRCS) $(TEST_EXTRACTION_SRCS) $(TEST_STORE_SRCS
702696

703697
BUILD_DIR = build/c
704698

705-
# Integration-asset hash header (variables defined next to PROJECT_HDRS; the
706-
# rule lives here because target lines expand $(BUILD_DIR) immediately).
707-
$(INTEGRATIONS_HASH_HDR): $(INTEGRATIONS_ASSET) scripts/gen-integrations-hash.sh | $(BUILD_DIR)
708-
sh scripts/gen-integrations-hash.sh $(INTEGRATIONS_ASSET) $@
709-
710699
# ── Object file compilation (grammars need relaxed warnings) ─────
711700

712701
# Grammar + tree-sitter runtime: compiled without -Werror (upstream code has warnings)
@@ -728,7 +717,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
728717

729718
# ── Targets ──────────────────────────────────────────────────────
730719

731-
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend ui-pack clean-c lint lint-tidy lint-cppcheck lint-format security
720+
.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security
732721

733722
$(BUILD_DIR):
734723
mkdir -p $(BUILD_DIR)
@@ -871,9 +860,9 @@ $(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUIL
871860
OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
872861
OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ)
873862

874-
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
863+
$(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
875864
$(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \
876-
$(ALL_TEST_SRCS) $(TEST_PROD_SRCS) \
865+
$(ALL_TEST_SRCS) $(PROD_SRCS) \
877866
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
878867
$(OBJS_VENDORED_TEST) \
879868
$(LDFLAGS_TEST)
@@ -899,9 +888,9 @@ test-focused: $(BUILD_DIR)/test-runner
899888
# Mirrors test-runner's link line but uses repro_main.c (own main + counters)
900889
# and TEST_REPRO_SRCS instead of ALL_TEST_SRCS. Exits non-zero while any bug is
901890
# still reproduced (the expected state); bug-repro.yml surfaces it as a board.
902-
$(BUILD_DIR)/test-repro-runner: $(TEST_REPRO_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
891+
$(BUILD_DIR)/test-repro-runner: $(TEST_REPRO_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) $(PROJECT_HDRS) | $(BUILD_DIR)
903892
$(CC) $(CFLAGS_TEST) -Itests -o $@ \
904-
$(TEST_REPRO_SRCS) $(TEST_PROD_SRCS) \
893+
$(TEST_REPRO_SRCS) $(PROD_SRCS) \
905894
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
906895
$(OBJS_VENDORED_TEST) \
907896
$(LDFLAGS_TEST)
@@ -959,9 +948,9 @@ TSAN_OPTIONS ?= halt_on_error=1:report_thread_leaks=0
959948
# Normal ASan/native/soak paths remain uncapped. High-worker TSan diagnostics
960949
# invoke test-runner-tsan directly so release gates cannot drift accidentally.
961950

962-
$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(TEST_PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) $(PROJECT_HDRS) | $(BUILD_DIR)
951+
$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) $(PROJECT_HDRS) | $(BUILD_DIR)
963952
$(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \
964-
$(ALL_TEST_SRCS) $(TEST_PROD_SRCS) \
953+
$(ALL_TEST_SRCS) $(PROD_SRCS) \
965954
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
966955
$(OBJS_VENDORED_TSAN) \
967956
$(LDFLAGS_TSAN)
@@ -1048,40 +1037,29 @@ $(BUILD_DIR)/codebase-memory-mcp: $(MAIN_SRC) $(PROD_SRCS) $(EXTRACTION_SRCS) $(
10481037
cbm: $(BUILD_DIR)/codebase-memory-mcp
10491038
@echo "Built: $(BUILD_DIR)/codebase-memory-mcp"
10501039

1051-
# ── Build with external UI asset pack (requires Node.js) ────────
1040+
# ── Build with embedded UI (requires Node.js) ───────────────────
10521041

1053-
# The native image contains only a generated manifest (pack basename, SHA-256
1054-
# and byte length). Frontend bytes stay in a content-addressed sidecar next to
1055-
# the binary so release scanners can inspect them independently.
1056-
UI_ASSET_MANIFEST = $(BUILD_DIR)/generated/ui_asset_manifest.c
1057-
UI_ASSET_DIST ?= graph-ui/dist
1058-
UI_ASSET_PREBUILT ?= 0
1059-
ifeq ($(UI_ASSET_PREBUILT),1)
1060-
UI_ASSET_BUILD_PREREQ :=
1061-
else
1062-
UI_ASSET_BUILD_PREREQ := frontend
1063-
endif
1064-
PROD_SRCS_WITH_ASSETS = $(subst src/ui/asset_pack_stub.c,src/ui/asset_pack.c $(UI_ASSET_MANIFEST),$(PROD_SRCS))
1042+
# Swap embedded_stub.c for the generated embedded_assets.c
1043+
UI_SRCS_WITH_ASSETS = $(subst src/ui/embedded_stub.c,src/ui/embedded_assets.c,$(UI_SRCS))
1044+
PROD_SRCS_WITH_ASSETS = $(subst src/ui/embedded_stub.c,src/ui/embedded_assets.c,$(PROD_SRCS))
1045+
1046+
# Embedded asset object files (generated by embed script)
1047+
EMBED_OBJS = $(wildcard $(BUILD_DIR)/embedded/embed_*.o)
10651048

10661049
frontend:
10671050
cd graph-ui && npm ci && npm run build
10681051

1069-
$(UI_ASSET_MANIFEST): $(UI_ASSET_BUILD_PREREQ) scripts/pack-ui-assets.mjs | $(BUILD_DIR)
1070-
node scripts/pack-ui-assets.mjs $(UI_ASSET_DIST) $(BUILD_DIR) $@
1071-
1072-
ui-pack: $(UI_ASSET_MANIFEST)
1052+
embed: frontend
1053+
scripts/embed-frontend.sh graph-ui/dist $(BUILD_DIR)/embedded
10731054

1074-
# $(INTEGRATIONS_HASH_HDR): generated header included by integration_assets.c.
1075-
# The standard target gets it via $(PROJECT_HDRS); the UI target compiles in one
1076-
# shot and must list it too, or the build fails with "cbm_integrations_hash.h
1077-
# file not found" (seen on the Windows UI build).
1078-
cbm-with-ui: ui-pack $(INTEGRATIONS_HASH_HDR) $(OBJS_VENDORED_PROD)
1055+
cbm-with-ui: embed $(OBJS_VENDORED_PROD)
10791056
$(CC) $(CFLAGS_PROD) -o $(BUILD_DIR)/codebase-memory-mcp \
10801057
$(MAIN_SRC) $(PROD_SRCS_WITH_ASSETS) \
10811058
$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
10821059
$(OBJS_VENDORED_PROD) \
1060+
$(wildcard $(BUILD_DIR)/embedded/embed_*.o) \
10831061
$(LDFLAGS)
1084-
@echo "Built with UI asset pack: $(BUILD_DIR)/codebase-memory-mcp"
1062+
@echo "Built with UI: $(BUILD_DIR)/codebase-memory-mcp"
10851063

10861064
clean-c: export BUILD_DIR := $(BUILD_DIR)
10871065
clean-c:
@@ -1114,7 +1092,7 @@ lint-tidy:
11141092
@$(CLANG_TIDY) --quiet $(LINT_SRCS) -- $(CFLAGS_COMMON) $(SYSROOT_FLAG)
11151093

11161094
# cppcheck: complementary analysis (config in .cppcheck)
1117-
lint-cppcheck: $(INTEGRATIONS_HASH_HDR)
1095+
lint-cppcheck:
11181096
@echo "=== cppcheck ==="
11191097
@$(CPPCHECK) --enable=warning,style,performance,portability \
11201098
--std=c11 --language=c \

0 commit comments

Comments
 (0)