Skip to content

Commit f2e97ff

Browse files
authored
Merge pull request DeusData#1341 from DeusData/ci/vt-zero-tolerance
ci(release): revert the VT gate to zero tolerance
2 parents 4730fe9 + 3e343a3 commit f2e97ff

6 files changed

Lines changed: 92 additions & 436 deletions

File tree

‎.github/workflows/release.yml‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -233,22 +233,18 @@ jobs:
233233
vt_api_key: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
234234
files: binaries/*
235235

236-
# VERSION + GH_TOKEN enable the pre-release ML false-positive tolerance:
237-
# the gate may downgrade a single-engine Microsoft "!ml" verdict to
238-
# TOLERATED, but only for -rc./-pre versions and only when hash-pinned
239-
# Defender ENDPOINT evidence is attached to the draft release
240-
# (defender-endpoint-verification.txt, from scripts/ci/av-endpoint-verify.sh).
236+
# ZERO tolerance: any detection on any artifact blocks the release.
237+
# False positives are resolved upstream (Microsoft false-positive
238+
# submission for the exact hashes, then re-run this job — no rebuild),
239+
# never by loosening this gate.
241240
- name: Wait for VirusTotal results
242241
env:
243242
VT_API_KEY: ${{ secrets.VIRUS_TOTAL_SCANNER_API_KEY }}
244243
VT_ANALYSIS: ${{ steps.virustotal.outputs.analysis }}
245-
VERSION: ${{ inputs.version }}
246-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
247244
run: scripts/ci/check-virustotal.sh
248245

249-
# Renders per-binary sha256 + VT links into the notes; when the gate
250-
# tolerated an ML false positive it states that honestly instead of
251-
# claiming "0 detections".
246+
# Renders per-binary sha256 + VT links into the notes; only reachable
247+
# after the zero-tolerance gate passed, so "0 detections" is verified.
252248
- name: Append VirusTotal scan links to release notes
253249
env:
254250
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎scripts/ci/append-vt-notes.sh‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,13 @@
11
#!/usr/bin/env bash
22
# Append the Security Verification section to the release notes: per-binary
3-
# sha256 + VirusTotal links, and — when the pre-release ML false-positive
4-
# tolerance fired (scripts/ci/check-virustotal.sh wrote /tmp/vt_tolerated.tsv)
5-
# — an honest annotation instead of a blanket "0 detections" claim.
3+
# sha256 + VirusTotal links. This step only runs after check-virustotal.sh
4+
# passed, and that gate is ZERO tolerance — any detection blocks the release —
5+
# so "0 detections" here is a verified statement, never an assumption.
66
# Expects: GH_TOKEN, VERSION; run from the verify job workspace (binaries/).
77
set -euo pipefail
88

9-
TOL=/tmp/vt_tolerated.tsv
10-
119
TABLE="\n\n## Security Verification\n\n"
12-
if [ -s "$TOL" ]; then
13-
N=$(wc -l < "$TOL" | tr -d ' ')
14-
TABLE+="All release binaries were scanned with 70+ antivirus engines. "
15-
TABLE+="**${N} of them currently show a single machine-learning heuristic detection** "
16-
TABLE+="(Microsoft \`Wacatac!ml\`) that we have verified to be a false positive: "
17-
TABLE+="an up-to-date Microsoft Defender endpoint — signatures updated at scan time, "
18-
TABLE+="real-time protection on — scans the identical bytes clean. The hash-pinned "
19-
TABLE+="evidence is attached to this release as \`defender-endpoint-verification.txt\`, "
20-
TABLE+="and a false-positive report for these exact hashes has been submitted to "
21-
TABLE+="Microsoft. See **Antivirus false positives on release binaries** in the notes "
22-
TABLE+="above for the full story and what we are doing about it.\n\n"
23-
else
24-
TABLE+="All release binaries scanned with 70+ antivirus engines — **0 detections**.\n\n"
25-
fi
10+
TABLE+="All release binaries scanned with 70+ antivirus engines — **0 detections**.\n\n"
2611
TABLE+="| Binary | SHA-256 | VirusTotal |\n"
2712
TABLE+="|--------|---------|------------|\n"
2813

@@ -34,13 +19,7 @@ for bin in binaries/codebase-memory-mcp-*; do
3419
label=$(echo "$name" | sed 's/^codebase-memory-mcp-//' | sed 's/\.exe$//')
3520
short="${sha256:0:20}..."
3621
vt_url="https://www.virustotal.com/gui/file/${sha256}/detection"
37-
if [ -s "$TOL" ] && grep -q "^${name} " "$TOL"; then
38-
completed=$(grep "^${name} " "$TOL" | cut -f6)
39-
cell="[1/${completed} ⚠️ ML false positive, endpoint-verified clean](${vt_url})"
40-
else
41-
cell="[0 detections ✅](${vt_url})"
42-
fi
43-
TABLE+="| \`${label}\` | \`${short}\` | ${cell} |\n"
22+
TABLE+="| \`${label}\` | \`${short}\` | [0 detections ✅](${vt_url}) |\n"
4423
done
4524

4625
CURRENT=$(gh release view "$VERSION" \

‎scripts/ci/av-endpoint-verify.sh‎

Lines changed: 0 additions & 153 deletions
This file was deleted.

‎scripts/ci/check-virustotal.sh‎

Lines changed: 3 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,10 @@
11
#!/usr/bin/env bash
22
# Wait for VirusTotal scans to complete and check results.
33
# Expects: VT_API_KEY, VT_ANALYSIS (comma-separated "file=URL" pairs)
4-
# Optional: VERSION (release tag) + GH_TOKEN — these enable the PRE-RELEASE
5-
# single-ML-detection tolerance described below.
64
set -euo pipefail
75

86
MIN_ENGINES=60
9-
rm -f /tmp/vt_gate_fail /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv
10-
11-
# ── Pre-release ML false-positive tolerance ──────────────────────────────────
12-
# A verdict may be downgraded from BLOCKED to TOLERATED only when ALL of:
13-
# 1. the version is a pre-release (-rc. / -pre / -alpha / -beta),
14-
# 2. every failing file was flagged by exactly ONE engine,
15-
# 3. that engine is Microsoft and the verdict is a machine-learning
16-
# heuristic (result string ends in "!ml") — never a signature name,
17-
# 4. an up-to-date Microsoft Defender ENDPOINT has scanned the identical
18-
# bytes clean, proven by a hash-pinned evidence file attached to the
19-
# draft release (defender-endpoint-verification.txt, produced by
20-
# scripts/ci/av-endpoint-verify.sh).
21-
#
22-
# Why this is sound: VT's "Microsoft" engine is a differently-configured
23-
# instance of the same engine that ships in Windows, and for our large
24-
# binaries it sits at an unstable ML decision boundary — across four release
25-
# cycles the Wacatac!ml flag appeared and disappeared with no code-side cause
26-
# (stripping, downloader removal and metadata changes each "fixed" it only
27-
# until the next build). The shipping product's verdict on identical bytes,
28-
# with signatures updated minutes before the scan, is the stronger statement
29-
# of Microsoft's own estimate. Multi-engine hits, signature-named verdicts,
30-
# non-Microsoft engines, timeouts and stable releases ALWAYS block.
31-
TOLERANCE_ELIGIBLE=false
32-
case "${VERSION:-}" in
33-
*-rc.* | *-pre* | *-alpha* | *-beta*) TOLERANCE_ELIGIBLE=true ;;
34-
esac
7+
rm -f /tmp/vt_gate_fail
358

369
echo "=== Waiting for VirusTotal scans to fully complete ==="
3710

@@ -95,22 +68,6 @@ print(f'{status},{malicious},{suspicious},{completed},{total}')
9568
echo "BLOCKED: $BASENAME flagged ($MALICIOUS malicious, $SUSPICIOUS suspicious / $COMPLETED engines)"
9669
echo " $URL"
9770
echo "FAIL" >> /tmp/vt_gate_fail
98-
# Record WHO flagged it so the tolerance can classify the failure.
99-
FLAG_INFO=$(echo "$RESULT" | python3 -c "
100-
import json, sys
101-
d = json.loads(sys.stdin.read())
102-
res = d.get('data', {}).get('attributes', {}).get('results', {})
103-
flagged = [(e, (r.get('result') or '?')) for e, r in res.items()
104-
if r.get('category') in ('malicious', 'suspicious')]
105-
engines = ';'.join(e for e, _ in flagged)
106-
verdicts = ';'.join(v for _, v in flagged)
107-
print(f'{len(flagged)}\t{engines}\t{verdicts}')
108-
" 2>/dev/null || printf 'parse-error\tparse-error\tparse-error\n')
109-
SHA256=$( (sha256sum "$FILE" 2>/dev/null || shasum -a 256 "$FILE" 2>/dev/null) \
110-
| awk 'NR==1{print $1}')
111-
[ -n "$SHA256" ] || SHA256="unknown"
112-
printf '%s\t%s\t%s\t%s\n' "$BASENAME" "$SHA256" "$FLAG_INFO" "$COMPLETED" \
113-
>> /tmp/vt_suspects.tsv
11471
else
11572
echo "OK: $BASENAME clean ($COMPLETED engines, 0 detections)"
11673
fi
@@ -127,76 +84,8 @@ print(f'{len(flagged)}\t{engines}\t{verdicts}')
12784
fi
12885
done
12986

130-
if [ ! -f /tmp/vt_gate_fail ]; then
131-
echo "=== All VirusTotal scans passed ==="
132-
exit 0
133-
fi
134-
135-
# ── Classify the failures against the tolerance conditions ──────────────────
136-
FAILS=$(wc -l < /tmp/vt_gate_fail | tr -d ' ')
137-
SUSPECTS=0
138-
[ -f /tmp/vt_suspects.tsv ] && SUSPECTS=$(wc -l < /tmp/vt_suspects.tsv | tr -d ' ')
139-
140-
TOLERABLE="$TOLERANCE_ELIGIBLE"
141-
# Timeouts and unparseable URLs produce FAIL lines with no suspect record —
142-
# those are never tolerable, and the counts diverging catches them.
143-
[ "$FAILS" -eq "$SUSPECTS" ] || TOLERABLE=false
144-
if [ "$TOLERABLE" = "true" ]; then
145-
while IFS=$'\t' read -r _name _sha nflagged engines verdicts _completed; do
146-
[ "$nflagged" = "1" ] || TOLERABLE=false
147-
[ "$engines" = "Microsoft" ] || TOLERABLE=false
148-
case "$verdicts" in
149-
*'!ml') ;;
150-
*) TOLERABLE=false ;;
151-
esac
152-
done < /tmp/vt_suspects.tsv
153-
fi
154-
155-
if [ "$TOLERABLE" != "true" ]; then
87+
if [ -f /tmp/vt_gate_fail ]; then
15688
echo "BLOCKED: One or more VirusTotal checks failed"
15789
exit 1
15890
fi
159-
160-
# ── Every failure matches the ML-FP pattern: require endpoint evidence ──────
161-
EVIDENCE=/tmp/defender-endpoint-verification.txt
162-
rm -f "$EVIDENCE"
163-
if ! gh release download "${VERSION:?}" \
164-
--pattern defender-endpoint-verification.txt \
165-
--output "$EVIDENCE" --repo "${GITHUB_REPOSITORY:-DeusData/codebase-memory-mcp}" \
166-
2>/dev/null; then
167-
echo "BLOCKED: all detections match the pre-release ML false-positive pattern"
168-
echo " (single engine, Microsoft, '!ml' verdict) but the release carries no"
169-
echo " Defender endpoint evidence. To verify Microsoft's shipping product"
170-
echo " against these exact bytes and attach the proof, run locally:"
171-
echo ""
172-
NAMES=$(cut -f1 /tmp/vt_suspects.tsv | sed 's/\.exe$//' | tr '\n' ' ')
173-
echo " scripts/ci/av-endpoint-verify.sh $VERSION $NAMES--upload"
174-
echo ""
175-
echo " then RE-RUN THIS JOB (Re-run failed jobs — it does not rebuild, so"
176-
echo " the bytes and hashes stay identical)."
177-
exit 1
178-
fi
179-
180-
MISSING=false
181-
while IFS=$'\t' read -r name sha _rest; do
182-
if grep -qE "^${sha}[[:space:]]+[^[:space:]]+[[:space:]]+CLEAN[[:space:]]*$" "$EVIDENCE"; then
183-
echo "ENDPOINT-VERIFIED: $name ($sha) clean on a current Defender endpoint"
184-
else
185-
echo "BLOCKED: $name ($sha) has no CLEAN endpoint-evidence line"
186-
MISSING=true
187-
fi
188-
done < /tmp/vt_suspects.tsv
189-
if [ "$MISSING" = "true" ]; then
190-
echo " Evidence file present but stale or incomplete — regenerate it with"
191-
echo " scripts/ci/av-endpoint-verify.sh (hashes must match this build)."
192-
exit 1
193-
fi
194-
195-
cp /tmp/vt_suspects.tsv /tmp/vt_tolerated.tsv
196-
echo "=== TOLERATED (pre-release ML false-positive policy) ==="
197-
sed -n 's/^# //p' "$EVIDENCE"
198-
echo "Every remaining engine reports clean; the detections above are Microsoft"
199-
echo "ML heuristics that Microsoft's own shipping Defender does not reproduce."
200-
echo "The release notes will state this transparently, with the evidence file"
201-
echo "attached as a release asset."
202-
exit 0
91+
echo "=== All VirusTotal scans passed ==="

0 commit comments

Comments
 (0)