Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion scripts/ci/check-virustotal.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@ echo "$VT_ANALYSIS" | tr ',' '\n' | while IFS= read -r entry; do
continue
fi

STATS=$(echo "$RESULT" | python3 -c "
# Line 1 is the CSV summary; any further lines name a flagging engine.
# The engine names are what a detection is actionable ON: the FP submission
# has to go to a specific vendor, and "is this the known-noisy one or a new
# family" is the first question asked. Reporting only counts sent the reader
# to the web UI for the one fact the gate already had in hand.
REPORT=$(echo "$RESULT" | python3 -c "
import json, sys
d = json.loads(sys.stdin.read())
attrs = d.get('data', {}).get('attributes', {})
Expand All @@ -51,8 +56,16 @@ harmless = stats.get('harmless', 0)
total = sum(stats.values())
completed = malicious + suspicious + undetected + harmless
print(f'{status},{malicious},{suspicious},{completed},{total}')
for engine, r in sorted((attrs.get('results') or {}).items()):
if r.get('category') in ('malicious', 'suspicious'):
label = r.get('result') or r.get('category')
version = r.get('engine_version') or '?'
updated = r.get('engine_update') or '?'
print(f'{engine} = {label} (engine {version}, defs {updated})')
" 2>/dev/null || echo "queued,0,0,0,0")

STATS=$(echo "$REPORT" | head -1)
DETECTIONS=$(echo "$REPORT" | tail -n +2)
STATUS=$(echo "$STATS" | cut -d',' -f1)
MALICIOUS=$(echo "$STATS" | cut -d',' -f2)
SUSPICIOUS=$(echo "$STATS" | cut -d',' -f3)
Expand All @@ -66,6 +79,16 @@ print(f'{status},{malicious},{suspicious},{completed},{total}')
fi
if [ "$MALICIOUS" -gt 0 ] || [ "$SUSPICIOUS" -gt 0 ]; then
echo "BLOCKED: $BASENAME flagged ($MALICIOUS malicious, $SUSPICIOUS suspicious / $COMPLETED engines)"
if [ -n "$DETECTIONS" ]; then
echo "$DETECTIONS" | while IFS= read -r detection; do
[ -n "$detection" ] && echo " detected by: $detection"
done
else
# Counts say something flagged but no engine is named: report that
# rather than printing nothing, so the gap is visible instead of
# looking like a detection-free block.
echo " detected by: <engine names unavailable in the analysis response>"
fi
echo " $URL"
echo "FAIL" >> /tmp/vt_gate_fail
else
Expand Down
38 changes: 37 additions & 1 deletion tests/test_vt_gate_zero_tolerance_contract.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,19 @@ mk_response clean 0 0
mk_response one_malicious 1 0
mk_response one_suspicious 0 1

# Same detection as one_malicious, but carrying the per-engine results a real
# analysis returns. Pins that the gate NAMES the engine that fired.
cat > "$FIX/responses/named_engine.json" <<'EOF'
{"data":{"attributes":{"status":"completed",
"stats":{"malicious":1,"suspicious":0,"undetected":60,"harmless":0},
"results":{
"Microsoft":{"category":"malicious","result":"Trojan:Win32/Wacatac.B!ml",
"engine_version":"1.1.24010.10","engine_update":"20260807"},
"Kaspersky":{"category":"undetected","result":null,
"engine_version":"22.0.1.28","engine_update":"20260807"}
}}}}
EOF

echo "release-bytes" > "$FIX/work/binaries/probe"

run_gate() { # version analysis-id -> echoes exit code
Expand All @@ -74,5 +87,28 @@ for version in v0.9.1 v0.9.1-rc.1 v0.9.1-pre1 v1.0.0-alpha.1; do
fail "1 suspicious must block on $version"
done

# A detection must NAME the engine that fired.
#
# The gate reported counts only ("1 malicious / 59 engines"), which is the one
# thing a reader cannot act on: a false-positive submission goes to a SPECIFIC
# vendor (and the WDSI developer path is a web form, so there is no API to guess
# with), and triage starts with "known-noisy engine, or a new family?". In the
# 2026-08-07 dry run three artifacts flagged and the logs could not say by what,
# so the answer had to be fetched by hand from the web UI afterwards.
[ "$(run_gate v0.9.1 named_engine)" != "0" ] || fail "named-engine detection must still block"
grep -q "Microsoft" "$FIX/last.log" || \
fail "gate must name the flagging engine (Microsoft) — got: $(cat "$FIX/last.log")"
grep -q "Trojan:Win32/Wacatac.B!ml" "$FIX/last.log" || \
fail "gate must report the detection label, not just the vendor"
# Engines that did NOT flag must not be listed, or the actionable line drowns.
if grep -q "detected by: Kaspersky" "$FIX/last.log"; then
fail "gate listed a non-flagging engine (Kaspersky) as a detection"
fi

# Naming engines must not weaken the gate: an analysis WITHOUT per-engine
# results still blocks on the counts alone.
[ "$(run_gate v0.9.1 one_malicious)" != "0" ] || \
fail "detection without engine detail must still block"

rm -f /tmp/vt_gate_fail
echo "PASS: VT gate is zero tolerance — every detection blocks on every version"
echo "PASS: VT gate is zero tolerance and names the engine that fired"
Loading