Merge branch 'main' into feat/pkl-call-extraction

Author: tthayer

.github/workflows/dry-run.yml

@@ -1,11 +1,23 @@
 # Manual trigger: test everything before pushing a release.
 # Each step can be skipped for faster iteration.
 #
-# Pipeline:  lint → test → build → smoke/soak   (sequential chain)
-# Security:  security-static + codeql-gate       (independent island)
+# Pipeline:  lint → test → build → smoke/soak → virustotal   (sequential chain)
+# Security:  security-static + codeql-gate                    (independent island)
 #
 # Security does NOT block lint/test/build/smoke/soak. All jobs must pass
 # for the overall workflow to be green.
+#
+# VirusTotal runs LAST and scans the same artifacts the release scans, so a
+# detection surfaces here instead of mid-release. It is the one dry-run job
+# whose verdict does not depend on our code: the same unchanged bytes can come
+# back clean one day and flagged the next, which is precisely why finding out
+# during a dry run is worth the wait.
+#
+# Two independent skips, because they answer different questions:
+#   skip_builds      no artifacts at all — "do lint and tests pass?"
+#   skip_virustotal  build + smoke + soak, no scan — "do the archives pass CI?"
+# The second exists because the scan costs API quota and polls for up to two
+# hours, which is pure noise while iterating on a build or smoke failure.
 name: Dry Run
 
 on:
@@ -20,7 +32,11 @@ on:
         type: boolean
         default: false
       skip_builds:
-        description: 'Skip build + smoke'
+        description: 'Skip build + smoke (also skips VirusTotal — nothing to scan)'
+        type: boolean
+        default: false
+      skip_virustotal:
+        description: 'Skip VirusTotal scan (still builds + smokes the archives)'
         type: boolean
         default: false
       soak_level:
@@ -84,3 +100,55 @@ jobs:
     with:
       duration_minutes: 10
       run_asan: ${{ inputs.soak_level == 'full' }}
+
+  # ── VirusTotal (final step) ────────────────────────────────────
+  # Mirrors the release workflow's scan so a dry run answers the question the
+  # release will ask, before we tag anything. The only difference is where the
+  # archives come from: the release pulls them off a published release, a dry
+  # run has none, so they come from the build job's artifacts — the SAME files,
+  # produced by the same canonical package-release.sh.
+  #
+  # Runs after smoke and soak deliberately. It is the slowest job (polling can
+  # take up to two hours) and the one most likely to be waiting on someone
+  # else's queue, so it must not delay the feedback that is actually about our
+  # code.
+  virustotal:
+    if: ${{ inputs.skip_builds != true && inputs.skip_virustotal != true && !cancelled() && needs.build.result == 'success' && needs.smoke.result != 'failure' && needs.soak.result != 'failure' }}
+    needs: [build, smoke, soak]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: binaries-*
+          merge-multiple: true
+          path: assets
+
+      # Same extraction the release verify job performs, so the bytes handed to
+      # VirusTotal here are the bytes a user would download and run.
+      - name: Extract archive artifacts
+        run: scripts/ci/extract-release-archives.sh assets binaries
+
+      - name: VirusTotal scan
+        uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0
+        id: virustotal
+        with:
+          vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
+          files: binaries/*
+
+      # Same ZERO-tolerance gate as the release. A detection here is a real
+      # finding even when it is a false positive: it is what the release gate
+      # will do, so the dry run should say so now. Resolve it upstream (vendor
+      # false-positive submission for the exact hashes, then re-run) — never by
+      # loosening this gate.
+      - name: Wait for VirusTotal results
+        env:
+          VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
+          VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }}
+        run: scripts/ci/check-virustotal.sh

Makefile.cbm

@@ -75,7 +75,8 @@ endif
 # CBM_BIND_TS_ALLOCATOR=1: bind the tree-sitter runtime to mimalloc (#424). Only
 # the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where
 # binding would create an alloc/free mismatch, so the guard is prod-only.
-CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) $(CFLAGS_EXTRA)
+CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \
+              $(GLOBAL_OVERRIDE_DEFINE) $(CFLAGS_EXTRA)
 CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2
 
 # Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)
@@ -386,9 +387,34 @@ UI_SRCS = \
 # derives it from the MI_OVERRIDE option); MI_OVERRIDE alone is not read by the
 # source — it is kept here only as this project's prod/test marker.
 MIMALLOC_SRC = vendored/mimalloc/src/static.c
+# Linux belongs on this list too (#1360). The "Unix relies on static-link-order
+# override" strategy described above never actually worked: mimalloc defines
+# malloc/free ONLY when MI_MALLOC_OVERRIDE is set (it gates alloc-override.c),
+# so with the define off there were no strong symbols for the link order to
+# prefer. Measured on the shipped v0.9.1-rc.1 artifacts — linux-arm64 glibc AND
+# musl-static both report 0/6 allocator-owned size classes, i.e. ordinary malloc
+# was going to libc and every purge/reclaim option set in cbm_mem_init applied
+# only to the bound sqlite/tree-sitter populations.
+#
+# The macOS objection does not transfer. That abort is specific to the two-level
+# namespace, where this binary's free becomes mi_free while system libraries keep
+# allocating from the system allocator, so a pointer crossing the boundary hits
+# "mi_free: invalid pointer". ELF is a flat namespace: interposition is
+# process-wide and all-or-nothing, which is the configuration mimalloc is
+# normally deployed in. macOS therefore stays off, deliberately and permanently.
+#
+# TWO defines, switched together in ONE place so they cannot drift:
+# MI_MALLOC_OVERRIDE reaches the mimalloc translation unit and turns the
+# override on; CBM_MEM_GLOBAL_OVERRIDE reaches OUR sources and tells the startup
+# audit what to expect. CBM_MEM_GLOBAL_OVERRIDE goes only into CFLAGS_PROD,
+# because MIMALLOC_CFLAGS_TEST compiles mimalloc with -DMI_OVERRIDE=0 and no
+# override define at all — a test binary has no global override BY CONSTRUCTION
+# and must never be told to expect one, or it warns on every run.
 MIMALLOC_OVERRIDE_DEFINE :=
-ifeq ($(IS_MINGW),yes)
+GLOBAL_OVERRIDE_DEFINE :=
+ifneq ($(filter yes,$(IS_MINGW) $(IS_LINUX)),)
 MIMALLOC_OVERRIDE_DEFINE := -DMI_MALLOC_OVERRIDE=1
+GLOBAL_OVERRIDE_DEFINE := -DCBM_MEM_GLOBAL_OVERRIDE=1
 endif
 MIMALLOC_CFLAGS = -std=c11 -O2 -w \
                   -Ivendored/mimalloc/include \

internal/cbm/zstd_store.c

@@ -7,12 +7,12 @@
 #include <stddef.h>
 #include <stdint.h>
 
-int cbm_zstd_compress(const char *src, int srcLen, char *dst, int dstCap, int level) {
-    size_t rc = ZSTD_compress(dst, (size_t)dstCap, src, (size_t)srcLen, level);
+int64_t cbm_zstd_compress(const char *src, size_t srcLen, char *dst, size_t dstCap, int level) {
+    size_t rc = ZSTD_compress(dst, dstCap, src, srcLen, level);
     if (ZSTD_isError(rc)) {
         return 0;
     }
-    return (int)rc;
+    return (int64_t)rc;
 }
 
 int64_t cbm_zstd_decompress(const char *src, size_t srcLen, char *dst, size_t dstCap) {
@@ -31,6 +31,6 @@ size_t cbm_zstd_frame_content_size(const char *src, size_t srcLen) {
     return (size_t)n;
 }
 
-size_t cbm_zstd_compress_bound(int inputSize) {
-    return ZSTD_compressBound((size_t)inputSize);
+size_t cbm_zstd_compress_bound(size_t inputSize) {
+    return ZSTD_compressBound(inputSize);
 }

internal/cbm/zstd_store.h

@@ -5,8 +5,10 @@
 #include <stdint.h>
 
 // Zstd compression at specified level (1=fast .. 22=best).
-// Returns compressed size on success, 0 on error.
-int cbm_zstd_compress(const char *src, int srcLen, char *dst, int dstCap, int level);
+// srcLen/dstCap are size_t for the same reason decompression's are: a >2 GiB
+// database would otherwise wrap through int and hand the encoder a capacity that
+// disagrees with the real buffer. Returns compressed size on success, 0 on error.
+int64_t cbm_zstd_compress(const char *src, size_t srcLen, char *dst, size_t dstCap, int level);
 
 // Zstd decompression. srcLen/dstCap are size_t so a >2 GiB destination capacity
 // is never truncated (a large DB artifact, or a crafted one, would otherwise
@@ -20,6 +22,6 @@ int64_t cbm_zstd_decompress(const char *src, size_t srcLen, char *dst, size_t ds
 size_t cbm_zstd_frame_content_size(const char *src, size_t srcLen);
 
 // Maximum compressed size bound for given input size.
-size_t cbm_zstd_compress_bound(int inputSize);
+size_t cbm_zstd_compress_bound(size_t inputSize);
 
 #endif // CBM_ZSTD_STORE_H

scripts/ci/extract-release-archives.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Extract the release archives into a flat directory of scannable files.
+#
+# Turns the per-platform artifacts (codebase-memory-mcp-<goos>-<goarch>.tar.gz /
+# .zip, produced by the canonical package-release.sh) into one directory holding
+# the actual executables, each renamed to its platform so a scan report names
+# something a human can act on. The install scripts are copied in beside them:
+# they ship in the release too, and a compromised installer matters as much as a
+# compromised binary.
+#
+# Usage: extract-release-archives.sh <archive-dir> <output-dir>
+#
+# Lives in scripts/ rather than inline in the workflow because the venue-parity
+# contract requires it: a venue may provision, plumb artifacts, or call a
+# canonical leg script, and this is leg logic. It also means the dry run and the
+# release can extract identically instead of drifting apart in two YAML copies.
+set -euo pipefail
+
+ARCHIVE_DIR="${1:?extract-release-archives: missing <archive-dir>}"
+OUT_DIR="${2:?extract-release-archives: missing <output-dir>}"
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+mkdir -p "$OUT_DIR"
+shopt -s nullglob
+
+# Counts BINARIES recovered from archives, deliberately not "files in OUT_DIR".
+# The install scripts below are copied in unconditionally, so a whole-directory
+# count is never zero and would have reported success on an empty input — a
+# clean VirusTotal run over nothing, which is the precise false green this
+# script exists to prevent. Caught by running it against an empty directory.
+extracted=0
+
+for archive in "$ARCHIVE_DIR"/*.tar.gz; do
+  name="$(basename "$archive" .tar.gz)"
+  tar -xzf "$archive" -C "$OUT_DIR" 2>/dev/null || true
+  if [ -f "$OUT_DIR/codebase-memory-mcp" ]; then
+    mv "$OUT_DIR/codebase-memory-mcp" "$OUT_DIR/${name}"
+    extracted=$((extracted + 1))
+  fi
+done
+
+for archive in "$ARCHIVE_DIR"/*.zip; do
+  name="$(basename "$archive" .zip)"
+  unzip -o "$archive" -d "$OUT_DIR" >/dev/null 2>&1 || true
+  if [ -f "$OUT_DIR/codebase-memory-mcp.exe" ]; then
+    mv "$OUT_DIR/codebase-memory-mcp.exe" "$OUT_DIR/${name}.exe"
+    extracted=$((extracted + 1))
+  fi
+done
+
+if [ "$extracted" -eq 0 ]; then
+  echo "FAIL: no binaries extracted from '$ARCHIVE_DIR' — a scan here would have"
+  echo "      reported clean without examining a single byte"
+  ls -la "$ARCHIVE_DIR" 2>/dev/null || true
+  exit 1
+fi
+
+# Shipped alongside the binaries, so scanned alongside them: a compromised
+# installer matters as much as a compromised executable.
+cp "$ROOT/install.sh" "$OUT_DIR/" 2>/dev/null || true
+cp "$ROOT/install.ps1" "$OUT_DIR/" 2>/dev/null || true
+
+ls -la "$OUT_DIR"
+total="$(find "$OUT_DIR" -maxdepth 1 -type f | wc -l | tr -d ' ')"
+echo "extracted $extracted binarie(s); $total file(s) queued for scanning"

scripts/smoke-test.sh

@@ -187,6 +187,58 @@ if ! echo "$OUTPUT" | grep -qE 'v?[0-9]+\.[0-9]+|dev'; then
 fi
 echo "OK"
 
+echo ""
+echo "=== Phase 1b: allocator override matches this platform's contract ==="
+# Asserts the SHIPPED binary's actual allocator wiring, per platform:
+#
+#   Windows, Linux -> ordinary malloc MUST reach mimalloc (all size classes
+#                     owned). If it does not, every purge/reclaim option in
+#                     cbm_mem_init is decoration and freed pages stay committed
+#                     — that is #581, which hid in production for months
+#                     precisely because nothing asserted it on a real artifact.
+#   macOS          -> it MUST NOT. Enabling the override there aborts on the
+#                     first pointer crossing the two-level-namespace boundary
+#                     ("mi_free: invalid pointer"), so "owned" here would mean
+#                     we shipped a binary that crashes on index.
+#
+# Both directions fail. A silent flip either way is a release blocker, which is
+# why this lives in smoke (real artifact, all platforms) and not only in a unit
+# test built from source.
+ALLOC_LOG=$("$BINARY" cli list_projects 2>&1 >/dev/null || true)
+case "$(uname -s)" in
+  Darwin)
+    if echo "$ALLOC_LOG" | grep -q 'mem.allocator.not_owned'; then
+      echo "FAIL: macOS emitted the not_owned WARNING; expected the by-design"
+      echo "      bound-populations INFO line (see #1360)"
+      exit 1
+    fi
+    if echo "$ALLOC_LOG" | grep -q 'mem.allocator.owned'; then
+      echo "FAIL: macOS reports ordinary malloc as allocator-owned. The override"
+      echo "      must stay OFF here: under the two-level namespace it aborts"
+      echo "      with 'mi_free: invalid pointer' on the first crossing pointer."
+      exit 1
+    fi
+    echo "OK: macOS serves ordinary malloc from the system allocator, no warning"
+    ;;
+  MINGW*|MSYS*|CYGWIN*|Linux)
+    if echo "$ALLOC_LOG" | grep -q 'mem.allocator.not_owned'; then
+      echo "FAIL: ordinary malloc does NOT reach mimalloc on $(uname -s)."
+      echo "      Allocator tuning is inert and freed pages will stay committed (#581/#1360)."
+      echo "$ALLOC_LOG" | grep 'mem.allocator' | head -2
+      exit 1
+    fi
+    if echo "$ALLOC_LOG" | grep -q 'mem.allocator.bound_populations_only'; then
+      echo "FAIL: $(uname -s) reports bound-populations-only; the global override"
+      echo "      is expected to be compiled in on this platform (#1360)."
+      exit 1
+    fi
+    echo "OK: ordinary malloc reaches the allocator on $(uname -s)"
+    ;;
+  *)
+    echo "SKIP: no allocator contract defined for $(uname -s)"
+    ;;
+esac
+
 if [ "$SMOKE_MODE" != "--agent-config-only" ]; then
 echo ""
 echo "=== Phase 2: index test project ==="
@@ -338,6 +390,64 @@ if [ "$TOTAL" -lt 1 ]; then
   echo "FAIL: search_graph for 'compute' returned 0 results"
   exit 1
 fi
+
+echo ""
+echo "=== Phase 3z: no tool duplicates its payload into structuredContent (#1375) ==="
+# Asserts on the SHIPPED artifact what the unit suite asserts from source: a
+# non-JSON payload must travel ONCE. It used to appear twice — content[0].text
+# plus an identical structuredContent.text — costing 2.05x the bytes on a large
+# query_graph, i.e. half the 10 MiB transport budget and double the tokens billed
+# to every LLM caller.
+#
+# In smoke as well as the unit suite because this is a WIRE-FORMAT property: it
+# is what a real client actually receives from the real binary, and a from-source
+# test cannot prove the released artifact behaves the same way.
+DUP_TOOLS=0
+DUP_CHECKED=0
+for TOOL_ARGS in "search_graph --project $PROJECT --name-pattern compute" \
+                 "search_code --project $PROJECT --query compute" \
+                 "get_architecture --project $PROJECT" \
+                 "index_status --project $PROJECT"; do
+  # shellcheck disable=SC2086
+  ENVELOPE=$("$BINARY" cli $TOOL_ARGS --json 2>/dev/null || true)
+  [ -z "$ENVELOPE" ] && continue
+  VERDICT=$(printf '%s' "$ENVELOPE" | python3 -c '
+import json,sys
+try:
+    d = json.loads(sys.stdin.read())
+except Exception:
+    print("skip"); raise SystemExit
+if d.get("isError"):
+    print("skip"); raise SystemExit
+content = d.get("content") or []
+text = content[0].get("text", "") if content else ""
+sc = d.get("structuredContent")
+if not isinstance(sc, dict):
+    print("no-structured"); raise SystemExit
+try:
+    payload_is_object = isinstance(json.loads(text), dict)
+except Exception:
+    payload_is_object = False
+if payload_is_object:
+    print("skip"); raise SystemExit
+print("dup" if sc.get("text") == text and text else "ok")
+')
+  case "$VERDICT" in
+    dup) echo "FAIL: $(echo "$TOOL_ARGS" | cut -d" " -f1) repeats its payload in structuredContent (#1375)"; DUP_TOOLS=$((DUP_TOOLS+1)) ;;
+    ok) DUP_CHECKED=$((DUP_CHECKED+1)) ;;
+    no-structured) echo "FAIL: $(echo "$TOOL_ARGS" | cut -d" " -f1) has no structuredContent object (outputSchema requires one)"; DUP_TOOLS=$((DUP_TOOLS+1)) ;;
+  esac
+done
+if [ "$DUP_TOOLS" -ne 0 ]; then
+  echo "FAIL: $DUP_TOOLS tool(s) duplicate their payload on the shipped binary"
+  exit 1
+fi
+if [ "$DUP_CHECKED" -eq 0 ]; then
+  echo "FAIL: no tool produced a non-JSON payload — this check proved nothing"
+  exit 1
+fi
+echo "OK: $DUP_CHECKED tool(s) deliver their payload exactly once"
+
 echo "OK: search_graph found $TOTAL result(s) for 'compute'"
 
 # 3b: trace_path — verify compute has callers

scripts/test.sh

@@ -230,6 +230,9 @@ bash "$ROOT/tests/test_parallel_harness_contract.sh"
 echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
 bash "$ROOT/tests/test_venue_parity_contract.sh"
 
+echo "=== Step 0k: spawn console-window contract (#1427) ==="
+bash "$ROOT/tests/test_spawn_no_window_contract.sh"
+
 # Verify compiler supports target arch
 verify_compiler "$CC"