Skip to content

Commit a388700

Browse files
authored
Merge pull request DeusData#1252 from DeusData/fix/daemon-release-hardening
fix: harden daemon release paths and verification
2 parents 97ce23f + e5c7941 commit a388700

129 files changed

Lines changed: 20482 additions & 267 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/_test.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ jobs:
253253
mingw-w64-clang-${{ matrix.pkg }}-clang
254254
mingw-w64-clang-${{ matrix.pkg }}-compiler-rt
255255
mingw-w64-clang-${{ matrix.pkg }}-zlib
256+
mingw-w64-clang-${{ matrix.pkg }}-python3
256257
mingw-w64-clang-${{ matrix.pkg }}-ccache
257258
make
258259
git

‎.github/workflows/pr.yml‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ jobs:
5959
# The paginated files endpoint remains filename-only for this gate.
6060
FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename')
6161
printf '%s\n' "$FILES"
62-
if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|scripts/build\.sh|scripts/smoke-test\.sh|scripts/env\.sh|Makefile\.cbm)'; then
62+
if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then
6363
echo "product=true" >> "$GITHUB_OUTPUT"
6464
else
6565
echo "product=false" >> "$GITHUB_OUTPUT"
@@ -70,8 +70,8 @@ jobs:
7070
# time (e.g. the Windows CreateProcess argv-quoting class that previously
7171
# only surfaced in the release dry run). The full broad/emulated smoke stays
7272
# in the dry run. Only product-code PRs pay this; every leg gates.
73-
# SMOKE_DOWNLOAD_URL is intentionally unset — the download/update phases of
74-
# smoke-test.sh self-skip; the core index/search/trace phases still run. ──
73+
# The maintained local wrappers add a race-free release fixture so the
74+
# download/checksum/install/update phases run here as well. ──
7575
pr-smoke:
7676
needs: [changes]
7777
if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
@@ -104,6 +104,9 @@ jobs:
104104
mingw-w64-clang-x86_64-ccache
105105
make
106106
coreutils
107+
curl
108+
zip
109+
unzip
107110
108111
# Verified compiler cache — content-keyed, stale hits impossible by
109112
# construction (see scripts/env.sh).
@@ -119,7 +122,7 @@ jobs:
119122
if: matrix.os == 'ubuntu-latest'
120123
run: |
121124
scripts/build.sh CC=gcc CXX=g++
122-
scripts/smoke-test.sh "$(pwd)/build/c/codebase-memory-mcp"
125+
scripts/smoke-local.sh "$(pwd)/build/c/codebase-memory-mcp"
123126
env:
124127
CCACHE_DIR: ${{ github.workspace }}/.ccache
125128
CCACHE_MAXSIZE: 1000M
@@ -129,7 +132,7 @@ jobs:
129132
run: |
130133
scripts/build.sh CC=cc CXX=c++
131134
codesign --sign - --force build/c/codebase-memory-mcp
132-
scripts/smoke-test.sh "$(pwd)/build/c/codebase-memory-mcp"
135+
scripts/smoke-local.sh "$(pwd)/build/c/codebase-memory-mcp"
133136
env:
134137
CCACHE_DIR: ${{ github.workspace }}/.ccache
135138
CCACHE_MAXSIZE: 1000M
@@ -139,19 +142,7 @@ jobs:
139142
shell: msys2 {0}
140143
run: |
141144
scripts/build.sh CC=clang CXX=clang++
142-
# MSYS2 /tmp is intentionally shared. The launcher correctly rejects
143-
# a bundle below that writable ancestor. RUNNER_TEMP is also below
144-
# the shared D:\a tree, so stage in the runner account's profile.
145-
PROFILE_ROOT="$(cygpath -u "$USERPROFILE")"
146-
SMOKE_DIR="$(mktemp -d "$PROFILE_ROOT/cbm-pr-smoke.XXXXXX")"
147-
trap 'rm -rf "$SMOKE_DIR"' EXIT
148-
cp build/c/codebase-memory-mcp-launcher.exe \
149-
"$SMOKE_DIR/codebase-memory-mcp.exe"
150-
cp build/c/codebase-memory-mcp.exe \
151-
"$SMOKE_DIR/codebase-memory-mcp.payload.exe"
152-
CBM_CACHE_DIR="$(cygpath -m "$SMOKE_DIR/cache")" \
153-
SMOKE_TEMP_ROOT="$SMOKE_DIR" \
154-
scripts/smoke-test.sh "$SMOKE_DIR/codebase-memory-mcp.exe"
145+
SMOKE_ARCH=amd64 bash test-infrastructure/vm/vm-smoke.sh
155146
env:
156147
CCACHE_DIR: ${{ github.workspace }}/.ccache
157148
CCACHE_MAXSIZE: 1000M

‎Makefile.cbm‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,31 @@ CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \
9494
# Auto-detected via compiler; no manual override needed.
9595
IS_MINGW := $(shell echo | $(CC) -dM -E - 2>/dev/null | grep -q 'define _WIN32 ' && echo yes || echo no)
9696
WIN32_LIBS :=
97+
# Route Windows allocations through mimalloc (#581). mimalloc's own static
98+
# override is gated on _MSC_VER, which clang/MinGW never defines, so it
99+
# compiled out and the CRT won the link: ordinary malloc went to the CRT heap,
100+
# which keeps freed pages committed, leaving every allocator tuning in
101+
# cbm_mem_init POSIX-only. --wrap redirects the same population of allocations
102+
# that link-order override captures on POSIX, so the purge configuration
103+
# finally applies here too. src/foundation/mem_override_win.c supplies the
104+
# wrappers; its deallocating paths check mi_is_in_heap_region so a CRT-owned
105+
# pointer can never reach mi_free (#424).
106+
MIMALLOC_WRAP_SYMS := malloc calloc realloc free strdup strndup _msize \
107+
_aligned_malloc _aligned_free
108+
MIMALLOC_WRAP_FLAGS :=
109+
ifeq ($(IS_MINGW),yes)
110+
MIMALLOC_WRAP_FLAGS := $(foreach sym,$(MIMALLOC_WRAP_SYMS),-Wl,--wrap=$(sym))
111+
endif
112+
# Linux wraps a smaller set for MEASUREMENT only (see mem_override_posix.c):
113+
# allocations already reach mimalloc here, but the profiler needs the same
114+
# observation point as Windows or the platform comparison is not like-for-like.
115+
# macOS ld has no --wrap, so it stays census-only.
116+
IS_LINUX := $(shell uname -s 2>/dev/null | grep -q Linux && echo yes || echo no)
117+
MIMALLOC_WRAP_SYMS_POSIX := malloc calloc realloc free strdup
118+
MIMALLOC_WRAP_FLAGS_POSIX :=
119+
ifeq ($(IS_LINUX),yes)
120+
MIMALLOC_WRAP_FLAGS_POSIX := $(foreach sym,$(MIMALLOC_WRAP_SYMS_POSIX),-Wl,--wrap=$(sym))
121+
endif
97122
ifeq ($(IS_MINGW),yes)
98123
WIN32_LIBS := -lws2_32 -lpsapi -lshell32 -ladvapi32 -Wl,--allow-multiple-definition -Wl,--stack,8388608 -static
99124
endif
@@ -103,13 +128,19 @@ ifeq ($(STATIC),1)
103128
STATIC_FLAGS := -static
104129
endif
105130

106-
LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS)
107-
LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS)
108-
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS)
131+
# The POSIX wrap shim exists only so the profiler can observe allocations. It
132+
# must never reach a sanitized build: the Linux/macOS test builds are CRT+ASan,
133+
# and redirecting malloc into mimalloc underneath ASan's own interception mixes
134+
# two allocators on the same pointers. Windows keeps its wrap flags everywhere,
135+
# because there the shim is what makes mimalloc own the allocations at all.
136+
LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) $(MIMALLOC_WRAP_FLAGS)
137+
LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS)
138+
LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) $(MIMALLOC_WRAP_FLAGS)
109139

110140
# ── Source files ─────────────────────────────────────────────────
111141

112142
FOUNDATION_SRCS = \
143+
src/foundation/mem_override_win.c \
113144
src/foundation/arena.c \
114145
src/foundation/hash_table.c \
115146
src/foundation/str_intern.c \
@@ -750,6 +781,11 @@ test-repro: $(BUILD_DIR)/test-repro-runner
750781
TEST_TSAN_SUITES ?= mem slab_alloc parallel worker_pool watcher httpd pipeline \
751782
diagnostics mcp mcp_mutation_guard subprocess daemon daemon_application
752783
TSAN_OPTIONS ?= halt_on_error=1
784+
# A fixed concurrent envelope keeps TSan deterministic across developer hosts
785+
# and GitHub runners. Four workers still exercise real races without turning
786+
# sanitizer-instrumented allocator bookkeeping into a high-core lock convoy.
787+
# Normal ASan/native/soak paths remain uncapped. High-worker TSan diagnostics
788+
# invoke test-runner-tsan directly so release gates cannot drift accidentally.
753789

754790
$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR)
755791
$(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \
@@ -759,7 +795,9 @@ $(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS)
759795
$(LDFLAGS_TSAN)
760796

761797
test-tsan: $(BUILD_DIR)/test-runner-tsan
762-
cd $(CURDIR) && TSAN_OPTIONS="$(TSAN_OPTIONS)" $(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES)
798+
@echo "ThreadSanitizer workers: 4"
799+
cd $(CURDIR) && CBM_WORKERS=4 TSAN_OPTIONS="$(TSAN_OPTIONS)" \
800+
$(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES)
763801

764802
# Real-binary POSIX lifecycle smoke. The endpoint is deliberately account-wide;
765803
# an explicit Make invocation requires a clean rendezvous and fails rather than

‎internal/cbm/cbm.c‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -262,21 +262,47 @@ static TSParser *get_thread_parser(const TSLanguage *ts_lang, CBMLanguage lang)
262262
* (sqlite requires 8-byte-aligned roundup, and mimalloc honors that alignment).
263263
* Field order matches struct sqlite3_mem_methods exactly:
264264
* xMalloc, xFree, xRealloc, xSize, xRoundup, xInit, xShutdown, pAppData. */
265+
/* Profiled: these bindings bypass the malloc interposer entirely, so without a
266+
* hook here the biggest per-request allocations in the process — SQLite's page
267+
* cache and its query working set — are invisible to the attribution profile
268+
* (#581). */
265269
static void *cbm_sqlite_malloc(int n) {
266-
return mi_malloc((size_t)n);
270+
void *block = mi_malloc((size_t)n);
271+
return block;
267272
}
268273
static void cbm_sqlite_free(void *p) {
269274
mi_free(p);
270275
}
271276
static void *cbm_sqlite_realloc(void *p, int n) {
272-
return mi_realloc(p, (size_t)n);
277+
if (p) {}
278+
void *grown = mi_realloc(p, (size_t)n);
279+
return grown;
273280
}
274281
static int cbm_sqlite_size(void *p) {
275282
return (int)mi_usable_size(p);
276283
}
277284
static int cbm_sqlite_roundup(int n) {
278285
return (n + 7) & ~7; /* round up to 8-byte boundary */
279286
}
287+
/* Same reasoning as the sqlite bindings: tree-sitter allocates its parse trees
288+
* through these, and they too skip the interposer. */
289+
static void *cbm_ts_malloc(size_t n) {
290+
void *block = mi_malloc(n);
291+
return block;
292+
}
293+
static void *cbm_ts_calloc(size_t count, size_t size) {
294+
void *block = mi_calloc(count, size);
295+
return block;
296+
}
297+
static void *cbm_ts_realloc(void *p, size_t n) {
298+
if (p) {}
299+
void *grown = mi_realloc(p, n);
300+
return grown;
301+
}
302+
static void cbm_ts_free(void *p) {
303+
mi_free(p);
304+
}
305+
280306
static int cbm_sqlite_meminit(void *appdata) {
281307
(void)appdata;
282308
return SQLITE_OK;
@@ -295,7 +321,7 @@ void cbm_alloc_init(void) {
295321
alloc_bound = 1;
296322

297323
/* tree-sitter runtime (was previously bound in cbm_init; consolidated here). */
298-
ts_set_allocator(mi_malloc, mi_calloc, mi_realloc, mi_free);
324+
ts_set_allocator(cbm_ts_malloc, cbm_ts_calloc, cbm_ts_realloc, cbm_ts_free);
299325

300326
/* sqlite3. SQLITE_CONFIG_MALLOC MUST run before sqlite3_initialize / the
301327
* first sqlite3_open* — otherwise sqlite3_config returns SQLITE_MISUSE

0 commit comments

Comments
 (0)