Merge pull request #18 from ngallodev/sync/upstream-step4

Sync with upstream codebase-memory-mcp (665 → 0 behind) via real merges, MCP features ported to CLI

Author: ngallodev

.github/workflows/_lint.yml

@@ -10,7 +10,11 @@ permissions:
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
+    # Pinned, not ubuntu-latest: the LLVM install below adds an
+    # apt.llvm.org llvm-toolchain-NOBLE repo, which is 24.04 by name. When
+    # GitHub moves ubuntu-latest to the next LTS this job breaks on a day
+    # nobody chose. Pin the runner to the release the repo line already names.
+    runs-on: ubuntu-24.04
     # 15 min proved too tight on a slow runner day and a timed-out lint reads
     # as "cancelled", which the release graph must treat as a hard stop; keep
     # a bound, but one only a genuine hang can hit (normal runtime ~5 min).
@@ -60,7 +64,8 @@ jobs:
   # provability (never suppressed — the NOLINT ban applies here too).
   # Vendored-tree diagnostics are path-filtered, mirroring .cppcheck.
   lint-mem:
-    runs-on: ubuntu-latest
+    # Pinned for the same reason as `lint`: llvm-toolchain-noble-21 below.
+    runs-on: ubuntu-24.04
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

.github/workflows/_memwaste.yml

@@ -0,0 +1,183 @@
+# Reusable: the waste sanitizer (memory + CPU waste), REPORT-ONLY.
+#
+# What runs (scripts/memwaste.sh, memwaste-scaling.py, memwaste-ratchet.py):
+#   mode=pr    Linux: the event lane (memory waste + CPU work) indexing this
+#              repository at the checked-out commit, and the CPU scaling lane
+#              (src/ replicated 2x against 4x, per-function counter ratios).
+#   mode=full  the same, plus the event lane on macOS and Windows and the access
+#              lane (read/write verdicts per allocation) on Linux.
+#
+# The corpus is the repository itself: pinned by the commit, no network, and a
+# real mix of C, C++, Python, JavaScript and vendored code.
+#
+# REPORT-ONLY: every job uploads its report and prints the ratchet comparison
+# against scripts/memwaste-baseline.json, but nothing here can turn a PR red --
+# the ratchet runs without --enforce and the PR's ci-ok gate does not depend on
+# these jobs. A job fails only when the MEASUREMENT is unsound (a table
+# overflowed, frees the layer never saw): that is a defect of the tool, and a
+# red job is how it gets noticed. Switching the ratchet to --enforce is a
+# separate, deliberate change once the numbers have been stable across reruns.
+name: Memwaste
+
+on:
+  workflow_call:
+    inputs:
+      mode:
+        description: "pr (Linux event + scaling) or full (all platforms, access lane)"
+        type: string
+        default: pr
+
+permissions:
+  contents: read
+
+jobs:
+  event-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install deps
+        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+      - name: Event lane on this repository
+        run: |
+          # `cmd | tee` reports TEE's status: without this a failing lane
+          # (a missing exec bit, a build error) passes the step and only
+          # surfaces as the ratchet not finding its JSON.
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP/memwaste-event"
+          scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
+            | tee "$RUNNER_TEMP/memwaste-event/report.txt"
+      - name: Ratchet (report-only)
+        run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue ${{ inputs.mode }}-linux
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: memwaste-event-linux-${{ inputs.mode }}
+          path: |
+            ${{ runner.temp }}/memwaste-event/report.txt
+            ${{ runner.temp }}/memwaste-event/memwaste.json
+            ${{ runner.temp }}/memwaste-event/index.json
+          if-no-files-found: warn
+
+  scaling-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install deps
+        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+      - name: CPU scaling lane (src/ at 2x against 4x)
+        # Report-only: the script exits 1 when it flags a super-linear function.
+        continue-on-error: true
+        run: |
+          # `cmd | tee` reports TEE's status: without this a failing lane
+          # (a missing exec bit, a build error) passes the step and only
+          # surfaces as the ratchet not finding its JSON.
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP/memwaste-scaling"
+          python3 scripts/memwaste-scaling.py --corpus src --replicas 2 \
+            --out "$RUNNER_TEMP/memwaste-scaling" | tee "$RUNNER_TEMP/memwaste-scaling/report.txt"
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: memwaste-scaling-linux-${{ inputs.mode }}
+          path: ${{ runner.temp }}/memwaste-scaling/report.txt
+          if-no-files-found: warn
+
+  access-linux:
+    if: ${{ inputs.mode == 'full' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install deps
+        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 clang llvm
+      - name: Access lane on this repository
+        run: |
+          # `cmd | tee` reports TEE's status: without this a failing lane
+          # (a missing exec bit, a build error) passes the step and only
+          # surfaces as the ratchet not finding its JSON.
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP/memwaste-access"
+          scripts/memwaste.sh "$GITHUB_WORKSPACE" --lane access --top 40 --sort untouched \
+            --out "$RUNNER_TEMP/memwaste-access" | tee "$RUNNER_TEMP/memwaste-access/report.txt"
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: memwaste-access-linux-${{ inputs.mode }}
+          path: |
+            ${{ runner.temp }}/memwaste-access/report.txt
+            ${{ runner.temp }}/memwaste-access/memwaste.json
+          if-no-files-found: warn
+
+  event-macos:
+    if: ${{ inputs.mode == 'full' }}
+    runs-on: macos-14
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Event lane on this repository
+        run: |
+          # `cmd | tee` reports TEE's status: without this a failing lane
+          # (a missing exec bit, a build error) passes the step and only
+          # surfaces as the ratchet not finding its JSON.
+          set -o pipefail
+          mkdir -p "$RUNNER_TEMP/memwaste-event"
+          scripts/memwaste.sh "$GITHUB_WORKSPACE" --top 40 --out "$RUNNER_TEMP/memwaste-event" \
+            | tee "$RUNNER_TEMP/memwaste-event/report.txt"
+      - name: Ratchet (report-only)
+        run: python3 scripts/memwaste-ratchet.py --out "$RUNNER_TEMP/memwaste-event" --venue full-macos
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: memwaste-event-macos-${{ inputs.mode }}
+          path: |
+            ${{ runner.temp }}/memwaste-event/report.txt
+            ${{ runner.temp }}/memwaste-event/memwaste.json
+          if-no-files-found: warn
+
+  event-windows:
+    if: ${{ inputs.mode == 'full' }}
+    runs-on: windows-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2
+        with:
+          msystem: CLANG64
+          path-type: inherit
+          install: >-
+            mingw-w64-clang-x86_64-clang
+            mingw-w64-clang-x86_64-llvm
+            mingw-w64-clang-x86_64-zlib
+            mingw-w64-clang-x86_64-python3
+            make
+            git
+            coreutils
+      - name: Event lane on this repository
+        shell: msys2 {0}
+        run: |
+          # `cmd | tee` reports TEE's status: without this a failing lane
+          # (a missing exec bit, a build error) passes the step and only
+          # surfaces as the ratchet not finding its JSON.
+          set -o pipefail
+          out="$(cygpath -u "$RUNNER_TEMP")/memwaste-event"
+          mkdir -p "$out"
+          CC=clang CXX=clang++ scripts/memwaste.sh "$(cygpath -u "$GITHUB_WORKSPACE")" --top 40 --out "$out" \
+            | tee "$out/report.txt"
+      - name: Ratchet (report-only)
+        shell: msys2 {0}
+        run: python3 scripts/memwaste-ratchet.py --out "$(cygpath -u "$RUNNER_TEMP")/memwaste-event" --venue full-windows
+      - name: Upload report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: memwaste-event-windows-${{ inputs.mode }}
+          path: |
+            ${{ runner.temp }}/memwaste-event/report.txt
+            ${{ runner.temp }}/memwaste-event/memwaste.json
+          if-no-files-found: warn

.github/workflows/_test.yml

@@ -257,7 +257,9 @@ jobs:
   # parallel suite — catches what an older compiler-rt can miss. Same
   # canonical scripts/test.sh wave as every other leg.
   test-diag:
-    runs-on: ubuntu-latest
+    # Pinned for the same reason as the lint jobs: the pinned diagnostic
+    # toolchain below installs from llvm-toolchain-noble-21.
+    runs-on: ubuntu-24.04
     timeout-minutes: 120
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

.github/workflows/codeql.yml

@@ -29,7 +29,7 @@ jobs:
         run: sudo apt-get update && sudo apt-get install -y zlib1g-dev
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
+        uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
         with:
           languages: c-cpp
           build-mode: manual
@@ -38,6 +38,6 @@ jobs:
         run: scripts/build.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
+        uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
         with:
           category: "/language:c-cpp"

.github/workflows/dry-run.yml

@@ -45,6 +45,10 @@ on:
         description: 'Skip candidate scan; mark unscanned and default to stripped'
         type: boolean
         default: false
+      skip_memwaste:
+        description: 'Skip the waste sanitizer (report-only) and the fuzz smoke'
+        type: boolean
+        default: false
       soak_level:
         description: 'Soak: full (quick+asan), quick (10min), none'
         type: choice
@@ -107,6 +111,38 @@ jobs:
     with:
       broad_platforms: true
 
+  # ── Waste sanitizer, REPORT-ONLY (independent island) ─────────
+  # Event lane on Linux, macOS and Windows, access lane and the per-function
+  # scaling lane on Linux. Uploads reports; fails only when the measurement is
+  # unsound (a defect of the tool itself).
+  memwaste:
+    if: ${{ inputs.skip_memwaste != true }}
+    uses: ./.github/workflows/_memwaste.yml
+    with:
+      mode: full
+
+  # ── Fuzz smoke (independent island) ───────────────────────────
+  # A fixed number of executions from a fixed seed per target, so the verdict is
+  # a function of the commit (O9). A crash here is a real defect in a parser
+  # that reads untrusted input; its reproducer is uploaded.
+  fuzz:
+    if: ${{ inputs.skip_memwaste != true }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install deps
+        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
+      - name: Fuzz smoke (extract, cypher, config)
+        run: scripts/fuzz.sh all --runs 20000 --seed 1
+      - name: Upload reproducers
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fuzz-artifacts
+          path: build/fuzz/artifacts-*/
+          if-no-files-found: ignore
+
   # ── Soak tests (optional, parallel with smoke) ────────────────
   soak:
     if: ${{ inputs.soak_level != 'none' && !cancelled() && needs.build.result != 'failure' && needs.build.result != 'skipped' }}

.github/workflows/nightly-soak.yml

@@ -35,6 +35,27 @@ permissions:
   contents: read
 
 jobs:
+  # Fuzz exploration: new inputs are the point here, so it runs for a time
+  # budget instead of a fixed count. NON-GATING like the soak; a crash uploads
+  # its reproducer, which becomes a fixed-seed regression input once fixed.
+  fuzz:
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install deps
+        run: sudo apt-get update && sudo apt-get install -y zlib1g-dev clang llvm libclang-rt-dev
+      - name: Fuzz exploration (30 min per target)
+        run: scripts/fuzz.sh all --max-total-time 1800 --seed 0
+      - name: Upload reproducers
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: fuzz-artifacts-nightly
+          path: build/fuzz/artifacts-*/
+          if-no-files-found: ignore
+
   soak:
     uses: ./.github/workflows/_soak.yml
     with:

.github/workflows/pr.yml

@@ -64,9 +64,29 @@ jobs:
         run: |
           # The full .diff endpoint rejects large-but-valid PRs at 20k lines.
           # The paginated files endpoint remains filename-only for this gate.
-          FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename')
+          if FILES=$(gh api --paginate "repos/$REPO/pulls/$PR/files?per_page=100" --jq '.[].filename'); then
+            echo "file list: pulls/files API"
+          else
+            # The files endpoint renders the diff server-side too, and answers
+            # 422 "this diff is taking too long to generate" for a PR that
+            # regenerates a vendored parser (#2246: a 42 MB sql/parser.c) --
+            # so does the compare endpoint. The PR merge ref's first parent is
+            # the base, so a name-only diff across it is the same file list.
+            # depth 2 + blob:none fetches commits and trees only (well under
+            # 1 MB, under a second); nothing from the PR is checked out or run.
+            echo "::notice::pulls/files API failed; file list taken from refs/pull/$PR/merge"
+            CHANGES_GIT="$RUNNER_TEMP/changes.git"
+            git init -q --bare "$CHANGES_GIT"
+            git --git-dir="$CHANGES_GIT" fetch -q --depth=2 --filter=blob:none \
+              "https://github.com/$REPO" "refs/pull/$PR/merge"
+            FILES=$(git --git-dir="$CHANGES_GIT" diff --name-only --no-renames FETCH_HEAD^1 FETCH_HEAD)
+          fi
           printf '%s\n' "$FILES"
-          if printf '%s\n' "$FILES" | grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)'; then
+          # A here-string, not a pipe: under this shell's pipefail, grep -q
+          # exits on its first match, printf dies of SIGPIPE once the list
+          # outgrows the pipe buffer, and the MATCH is then reported as a
+          # failure -- product=false, smoke skipped, on the largest PRs.
+          if grep -qE '^(src/|internal/|install\.(sh|ps1)|scripts/build\.sh|scripts/smoke-test\.sh|scripts/smoke-local\.sh|scripts/smoke-fixture-server\.py|scripts/gen-third-party-notices\.sh|scripts/env\.sh|test-infrastructure/vm/(vm-smoke\.sh|windows-user-path-guard\.ps1)|Makefile\.cbm)' <<<"$FILES"; then
             echo "product=true" >> "$GITHUB_OUTPUT"
           else
             echo "product=false" >> "$GITHUB_OUTPUT"
@@ -154,6 +174,17 @@ jobs:
           CCACHE_DIR: ${{ github.workspace }}/.ccache
           CCACHE_MAXSIZE: 1000M
 
+  # ── Waste sanitizer, REPORT-ONLY: memory + CPU waste on this repository and
+  #    the per-function scaling lane (Linux). Deliberately NOT in ci-ok's needs:
+  #    it uploads evidence and prints the ratchet, it never blocks a merge. Only
+  #    product-code PRs pay for it. ──
+  memwaste:
+    needs: [changes]
+    if: ${{ !cancelled() && needs.changes.outputs.product == 'true' }}
+    uses: ./.github/workflows/_memwaste.yml
+    with:
+      mode: pr
+
   ci-ok:
     # The one required context (besides dco) — fails unless every PR stage
     # succeeded, so matrix renames can never silently deadlock merges. `skipped`