Skip to content

Commit d150ebe

Browse files
authored
Break-glass merge PR DeusData#1596
2 parents 973cde8 + 7b4533b commit d150ebe

26 files changed

Lines changed: 4174 additions & 729 deletions

‎.github/workflows/_build.yml‎

Lines changed: 286 additions & 54 deletions
Large diffs are not rendered by default.

‎.github/workflows/_soak.yml‎

Lines changed: 124 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ on:
2020
description: 'Version string for build'
2121
type: string
2222
default: ''
23+
use_release_artifacts:
24+
description: 'Soak the already selected binaries-* artifacts instead of rebuilding quick-lane binaries'
25+
type: boolean
26+
default: false
2327

2428
permissions:
2529
contents: read
@@ -64,12 +68,33 @@ jobs:
6468
if: startsWith(matrix.os, 'ubuntu')
6569
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev python3 git
6670
- name: Build
71+
if: ${{ !inputs.use_release_artifacts }}
6772
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
73+
- name: Download selected release container
74+
if: ${{ inputs.use_release_artifacts }}
75+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
76+
with:
77+
name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
78+
path: ${{ runner.temp }}/cbm-selected-container
79+
- name: Extract selected release executable
80+
if: ${{ inputs.use_release_artifacts }}
81+
run: |
82+
mkdir -p "$RUNNER_TEMP/cbm-selected-artifact"
83+
tar -xzf "$RUNNER_TEMP/cbm-selected-container/codebase-memory-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz" \
84+
-C "$RUNNER_TEMP/cbm-selected-artifact" codebase-memory-mcp
85+
chmod +x "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp"
86+
"$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" --version
6887
# The soak SEQUENCE and its guards live in the ONE canonical entry
6988
# scripts/soak-legs.sh — never in workflow YAML. Leg selection is a
7089
# workflow input so a long #581-only run needs no separate workflow.
7190
- name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
72-
run: scripts/soak-legs.sh --legs "${{ inputs.legs }}" build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
91+
env:
92+
SELECTED_BINARY: ${{ runner.temp }}/cbm-selected-artifact/codebase-memory-mcp
93+
USE_RELEASE_ARTIFACTS: ${{ inputs.use_release_artifacts }}
94+
run: |
95+
binary=build/c/codebase-memory-mcp
96+
test "$USE_RELEASE_ARTIFACTS" != true || binary="$SELECTED_BINARY"
97+
scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
7398
- name: Upload metrics
7499
if: always()
75100
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -104,6 +129,7 @@ jobs:
104129
make
105130
git
106131
coreutils
132+
unzip
107133
# Same protected root the test legs and the VM soak use. Without it the
108134
# soak ran under the runner's inherited LocalAppData\Temp ACLs while the
109135
# local soak ran hardened — the two venues were soaking different shapes.
@@ -115,8 +141,24 @@ jobs:
115141
-ProtectDir (Join-Path $env:GITHUB_WORKSPACE "build\c")
116142
"CBM_CI_TEMP_ROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
117143
- name: Build
144+
if: ${{ !inputs.use_release_artifacts }}
118145
shell: msys2 {0}
119146
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
147+
- name: Download selected release container
148+
if: ${{ inputs.use_release_artifacts }}
149+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
150+
with:
151+
name: binaries-windows-amd64
152+
path: ${{ runner.temp }}/cbm-selected-container
153+
- name: Extract selected release executable
154+
if: ${{ inputs.use_release_artifacts }}
155+
shell: msys2 {0}
156+
run: |
157+
ARTIFACT_DIR="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact"
158+
mkdir -p "$ARTIFACT_DIR"
159+
unzip -q "$(cygpath -u "$RUNNER_TEMP")/cbm-selected-container/codebase-memory-mcp-windows-amd64.zip" \
160+
codebase-memory-mcp.exe -d "$ARTIFACT_DIR"
161+
"$ARTIFACT_DIR/codebase-memory-mcp.exe" --version
120162
# Both release-gating legs via the ONE canonical soak entry; only the
121163
# protected-TEMP provisioning is venue text. soak-legs.sh resolves .exe.
122164
- name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
@@ -125,7 +167,11 @@ jobs:
125167
export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")"
126168
export TMP="$TEMP"
127169
export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")"
128-
scripts/soak-legs.sh --legs "${{ inputs.legs }}" build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
170+
binary=build/c/codebase-memory-mcp
171+
if [ "${{ inputs.use_release_artifacts }}" = true ]; then
172+
binary="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact/codebase-memory-mcp.exe"
173+
fi
174+
scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
129175
- name: Upload metrics
130176
if: always()
131177
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -161,6 +207,7 @@ jobs:
161207
make
162208
git
163209
coreutils
210+
unzip
164211
# Same protected root the test legs and the VM soak use. Without it the
165212
# soak ran under the runner's inherited LocalAppData\Temp ACLs while the
166213
# local soak ran hardened — the two venues were soaking different shapes.
@@ -172,8 +219,24 @@ jobs:
172219
-ProtectDir (Join-Path $env:GITHUB_WORKSPACE "build\c")
173220
"CBM_CI_TEMP_ROOT=$root" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
174221
- name: Build
222+
if: ${{ !inputs.use_release_artifacts }}
175223
shell: msys2 {0}
176224
run: scripts/build.sh ${{ inputs.version && format('--version {0}', inputs.version) || '' }} CC=clang CXX=clang++
225+
- name: Download selected release container
226+
if: ${{ inputs.use_release_artifacts }}
227+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
228+
with:
229+
name: binaries-windows-arm64
230+
path: ${{ runner.temp }}/cbm-selected-container
231+
- name: Extract selected release executable
232+
if: ${{ inputs.use_release_artifacts }}
233+
shell: msys2 {0}
234+
run: |
235+
ARTIFACT_DIR="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact"
236+
mkdir -p "$ARTIFACT_DIR"
237+
unzip -q "$(cygpath -u "$RUNNER_TEMP")/cbm-selected-container/codebase-memory-mcp-windows-arm64.zip" \
238+
codebase-memory-mcp.exe -d "$ARTIFACT_DIR"
239+
"$ARTIFACT_DIR/codebase-memory-mcp.exe" --version
177240
# Both release-gating legs via the ONE canonical soak entry; only the
178241
# protected-TEMP provisioning is venue text. soak-legs.sh resolves .exe.
179242
- name: Soak legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
@@ -182,7 +245,11 @@ jobs:
182245
export TEMP="$(cygpath -m "$CBM_CI_TEMP_ROOT")"
183246
export TMP="$TEMP"
184247
export TMPDIR="$(cygpath -u "$CBM_CI_TEMP_ROOT")"
185-
scripts/soak-legs.sh --legs "${{ inputs.legs }}" build/c/codebase-memory-mcp ${{ inputs.duration_minutes }}
248+
binary=build/c/codebase-memory-mcp
249+
if [ "${{ inputs.use_release_artifacts }}" = true ]; then
250+
binary="$(cygpath -u "$RUNNER_TEMP")/cbm-selected-artifact/codebase-memory-mcp.exe"
251+
fi
252+
scripts/soak-legs.sh --legs "${{ inputs.legs }}" "$binary" ${{ inputs.duration_minutes }}
186253
- name: Upload metrics
187254
if: always()
188255
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -198,7 +265,61 @@ jobs:
198265
path: soak-results-query-leak/
199266
retention-days: 14
200267

268+
# Portable Linux is a separate immutable product tuple. It participates when
269+
# a caller supplies selected release artifacts; the source-building nightly
270+
# retains its historical matrix and has no upstream binaries-* artifacts.
271+
soak-quick-linux-portable:
272+
if: ${{ inputs.use_release_artifacts }}
273+
strategy:
274+
fail-fast: false
275+
matrix:
276+
include:
277+
- arch: amd64
278+
runner: ubuntu-latest
279+
- arch: arm64
280+
runner: ubuntu-24.04-arm
281+
runs-on: ${{ matrix.runner }}
282+
timeout-minutes: 560
283+
steps:
284+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
285+
- name: Install soak dependencies
286+
run: sudo apt-get update && sudo apt-get install -y python3 git
287+
- name: Download selected portable release container
288+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
289+
with:
290+
name: binaries-linux-${{ matrix.arch }}-portable
291+
path: ${{ runner.temp }}/cbm-selected-container
292+
- name: Extract selected portable executable
293+
run: |
294+
mkdir -p "$RUNNER_TEMP/cbm-selected-artifact"
295+
tar -xzf "$RUNNER_TEMP/cbm-selected-container/codebase-memory-mcp-linux-${{ matrix.arch }}-portable.tar.gz" \
296+
-C "$RUNNER_TEMP/cbm-selected-artifact" codebase-memory-mcp
297+
chmod +x "$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp"
298+
"$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" --version
299+
- name: Soak selected portable legs (${{ inputs.legs }}, ${{ inputs.duration_minutes }} min each)
300+
run: |
301+
scripts/soak-legs.sh --legs "${{ inputs.legs }}" \
302+
"$RUNNER_TEMP/cbm-selected-artifact/codebase-memory-mcp" \
303+
${{ inputs.duration_minutes }}
304+
- name: Upload metrics
305+
if: always()
306+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
307+
with:
308+
name: soak-quick-linux-${{ matrix.arch }}-portable
309+
path: soak-results/
310+
retention-days: 14
311+
- name: Upload query-leak metrics
312+
if: always()
313+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
314+
with:
315+
name: soak-query-leak-linux-${{ matrix.arch }}-portable
316+
path: soak-results-query-leak/
317+
retention-days: 14
318+
201319
soak-asan:
320+
# Supplemental source-built instrumentation. This can find memory bugs but
321+
# is not evidence that the selected release bytes were soaked; the quick
322+
# jobs above own that release gate when use_release_artifacts=true.
202323
if: ${{ inputs.run_asan }}
203324
strategy:
204325
fail-fast: false

‎.github/workflows/dry-run.yml‎

Lines changed: 16 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,25 @@
11
# Manual trigger: test everything before pushing a release.
22
# Each step can be skipped for faster iteration.
33
#
4-
# Pipeline: lint → test → build → smoke/soak → virustotal (sequential chain)
4+
# Pipeline: lint → test → build + candidate scan/select → smoke/soak
55
# Security: security-static + codeql-gate (independent island)
66
#
77
# Security does NOT block lint/test/build/smoke/soak. All jobs must pass
88
# for the overall workflow to be green.
99
#
10-
# VirusTotal runs LAST and scans the same artifacts the release scans, so a
11-
# detection surfaces here instead of mid-release. It is the one dry-run job
12-
# whose verdict does not depend on our code: the same unchanged bytes can come
13-
# back clean one day and flagged the next, which is precisely why finding out
14-
# during a dry run is worth the wait.
10+
# Candidate VirusTotal scanning and tuple-local selection are part of the
11+
# reusable build boundary. Smoke and soak therefore start only after all 16
12+
# stripped/unstripped candidates have completed and the selected exact bytes
13+
# have been packaged under the canonical artifact names.
1514
#
1615
# The skips are independent because they answer different questions:
17-
# skip_builds no artifacts at all "do lint and tests pass?"
18-
# skip_smoke build + scan, no smoke "does this diff scan red?"
19-
# skip_virustotal build + smoke + soak, no scan "do the archives pass CI?"
16+
# skip_builds no artifacts at all "do lint and tests pass?"
17+
# skip_smoke build + scan/select, no smoke "does this diff scan red?"
18+
# skip_virustotal default-stripped + smoke, no scan "do the archives pass CI?"
2019
#
21-
# skip_virustotal exists because the scan costs API quota and polls for up to
22-
# two hours, which is pure noise while iterating on a build or smoke failure.
23-
#
24-
# skip_smoke is the opposite trip: go straight from build to VirusTotal when the
25-
# question is only whether a diff turns the scan red. Smoke is in the VirusTotal
26-
# job's needs, so without this a smoke failure would withhold exactly the answer
27-
# such a run was dispatched to get.
20+
# skip_virustotal is a conspicuous dry-run-only bypass for API-quota-sensitive
21+
# iteration. It records unscanned-dry-run and deterministically chooses stripped
22+
# within each tuple. release.yml has no corresponding bypass.
2823
name: Dry Run
2924

3025
on:
@@ -43,11 +38,11 @@ on:
4338
type: boolean
4439
default: false
4540
skip_smoke:
46-
description: 'Skip smoke (build straight to VirusTotal)'
41+
description: 'Skip smoke (candidate scan/select still runs inside build)'
4742
type: boolean
4843
default: false
4944
skip_virustotal:
50-
description: 'Skip VirusTotal scan (still builds + smokes the archives)'
45+
description: 'Skip candidate scan; mark unscanned and default to stripped'
5146
type: boolean
5247
default: false
5348
soak_level:
@@ -91,6 +86,8 @@ jobs:
9186
uses: ./.github/workflows/_build.yml
9287
with:
9388
attest: false
89+
scan_candidates: ${{ !inputs.skip_virustotal }}
90+
secrets: inherit
9491

9592
# ── Smoke test every binary ────────────────────────────────────
9693
# Run unless builds were skipped or a build leg FAILED. Every build leg
@@ -111,77 +108,4 @@ jobs:
111108
with:
112109
duration_minutes: 10
113110
run_asan: ${{ inputs.soak_level == 'full' }}
114-
115-
# ── VirusTotal (final step) ────────────────────────────────────
116-
# Mirrors the release workflow's scan so a dry run answers the question the
117-
# release will ask, before we tag anything. The only difference is where the
118-
# archives come from: the release pulls them off a published release, a dry
119-
# run has none, so they come from the build job's artifacts — the SAME files,
120-
# produced by the same canonical package-release.sh.
121-
#
122-
# Runs after smoke and soak deliberately. It is the slowest job (polling can
123-
# take up to two hours) and the one most likely to be waiting on someone
124-
# else's queue, so it must not delay the feedback that is actually about our
125-
# code.
126-
#
127-
# It needs smoke and soak only so it runs LAST, not because it depends on
128-
# them: a skipped smoke or soak leaves the scan reachable (result 'skipped',
129-
# which is not 'failure'), while a FAILED one withholds it — if the artifacts
130-
# are broken, a verdict on them is not worth two hours of polling.
131-
virustotal:
132-
if: ${{ inputs.skip_builds != true && inputs.skip_virustotal != true && !cancelled() && needs.build.result == 'success' && needs.smoke.result != 'failure' && needs.soak.result != 'failure' }}
133-
needs: [build, smoke, soak]
134-
runs-on: ubuntu-latest
135-
permissions:
136-
contents: read
137-
steps:
138-
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
139-
with:
140-
persist-credentials: false
141-
142-
- name: Download build artifacts
143-
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
144-
with:
145-
pattern: binaries-*
146-
merge-multiple: true
147-
path: ${{ runner.temp }}/release-archives
148-
149-
# Same extraction the release verify job performs. VirusTotal receives
150-
# only extracted files and unpacked UI assets, never .tar.gz/.zip containers.
151-
- name: Extract archive artifacts
152-
run: |
153-
scripts/ci/extract-release-archives.sh "$RUNNER_TEMP/release-archives" binaries \
154-
--expect-archives=8 \
155-
--expect-binaries=8 \
156-
--expect-runtime-files=24
157-
158-
- name: VirusTotal scan
159-
uses: crazy-max/ghaction-virustotal@936d8c5c00afe97d3d9a1af26d017cfdf26800a2 # v5.0.0
160-
id: virustotal
161-
with:
162-
vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
163-
files: binaries/objects/*
164-
request_rate: 4
165-
166-
# Same ZERO-tolerance gate as the release. Every extracted object must arrive
167-
# naturally clean; any malicious or suspicious verdict fails this run.
168-
- name: Wait for VirusTotal results
169-
env:
170-
VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
171-
VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }}
172-
VT_EXPECTED_SCAN_SET: binaries/scan-set.tsv
173-
VT_ASSOCIATIONS: binaries/associations.tsv
174-
VT_RESULTS_PATH: binaries/vt-results.tsv
175-
MIN_ENGINES: 50
176-
run: scripts/ci/check-virustotal.sh
177-
178-
- name: Preserve VirusTotal scope and result evidence
179-
if: ${{ always() }}
180-
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
181-
with:
182-
name: virustotal-evidence-dry-run-${{ github.run_id }}
183-
path: |
184-
binaries/associations.tsv
185-
binaries/scan-set.tsv
186-
binaries/vt-results.tsv
187-
if-no-files-found: warn
111+
use_release_artifacts: true

0 commit comments

Comments
 (0)