|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Withhold the selected executables from the full-surface VirusTotal pass. |
| 3 | +# |
| 4 | +# WHY (measured, not assumed). The verify pass used to submit every extracted |
| 5 | +# object, selected executables included, on the stated grounds that "VirusTotal |
| 6 | +# is content-addressed, so identical bytes return the analysis it already holds |
| 7 | +# instead of re-running 70+ engines". That is not what happens. On the v0.10.5 |
| 8 | +# release all EIGHT re-submissions produced a NEW analysis: same VirusTotal |
| 9 | +# file-id, a timestamp 47 minutes later. |
| 10 | +# |
| 11 | +# candidate: file-id=2c00f485... ts=1786795957 (12:12:37Z) |
| 12 | +# verify : file-id=2c00f485... ts=1786798758 (12:59:18Z) |
| 13 | +# |
| 14 | +# Re-analysing identical bytes re-rolls a probabilistic classifier, and it |
| 15 | +# answered differently in the same hour, in both directions: |
| 16 | +# |
| 17 | +# 82750cd1 (linux-amd64) microsoft-ml -> clean |
| 18 | +# 6d3c5be6 (darwin-arm64) clean -> microsoft-ml |
| 19 | +# |
| 20 | +# The published release notes then cited the candidate verdict while the linked |
| 21 | +# page showed the verify verdict, so the table said "clean" for a binary |
| 22 | +# VirusTotal was flagging and vice versa. Nothing about the binaries differed: |
| 23 | +# same sha256 in both scans. |
| 24 | +# |
| 25 | +# The second scan also proved nothing the first did not. Identity is already |
| 26 | +# established by hash: verify-release-selection.py reconciles every published |
| 27 | +# container to the selected bytes before this step runs, and checksums.txt binds |
| 28 | +# the same digests publicly. A re-scan adds no assurance and one more roll. |
| 29 | +# |
| 30 | +# What still gets scanned is everything the candidate pass never saw: install.sh, |
| 31 | +# install.ps1, LICENSE, THIRD_PARTY_NOTICES.md, the MCPB manifest.json and the |
| 32 | +# unpacked UI assets. install.sh and install.ps1 are the highest-consequence |
| 33 | +# non-executable bytes we publish - users pipe them straight into a shell - and |
| 34 | +# that coverage is untouched. |
| 35 | +# |
| 36 | +# Fails closed: if nothing matches, the selection binding is broken; if nothing |
| 37 | +# is left, the surface scan would silently become a no-op. |
| 38 | +# |
| 39 | +# Usage: exclude-rescanned-selected-objects.sh <objects-dir> <release-selection.tsv> [manifest-out] |
| 40 | +set -euo pipefail |
| 41 | + |
| 42 | +OBJECTS_DIR="${1:?usage: exclude-rescanned-selected-objects.sh <objects-dir> <selection.tsv> [manifest-out]}" |
| 43 | +SELECTION="${2:?usage: exclude-rescanned-selected-objects.sh <objects-dir> <selection.tsv> [manifest-out]}" |
| 44 | +MANIFEST="${3:-}" |
| 45 | + |
| 46 | +test -d "$OBJECTS_DIR" || { echo "error: no objects directory: $OBJECTS_DIR" >&2; exit 1; } |
| 47 | +test -s "$SELECTION" || { echo "error: no selection evidence: $SELECTION" >&2; exit 1; } |
| 48 | + |
| 49 | +head -n 1 "$SELECTION" | grep -qx '# cbm-release-selection-v1' || { |
| 50 | + echo "error: wrong evidence marker in $SELECTION" >&2; exit 1; } |
| 51 | + |
| 52 | +# selected_sha256 is the 4th column; skip the '#' metadata block and the header. |
| 53 | +selected="$(mktemp)" |
| 54 | +withheld="$(mktemp)" |
| 55 | +trap 'rm -f "$selected" "$withheld"' EXIT |
| 56 | +# `|| true` on the grep: with `set -o pipefail` a zero-match grep would abort the |
| 57 | +# script here, before the explicit check below could say why. A guard that exits |
| 58 | +# silently is not a guard. |
| 59 | +awk -F '\t' '/^#/ {next} $1=="target" {next} {print $4}' "$SELECTION" \ |
| 60 | + | { grep -E '^[0-9a-f]{64}$' || true; } | sort -u > "$selected" |
| 61 | +if [ ! -s "$selected" ]; then |
| 62 | + echo "error: no selected sha256 values in $SELECTION — the selection" >&2 |
| 63 | + echo " evidence names no shipped bytes, so nothing can be matched." >&2 |
| 64 | + exit 1 |
| 65 | +fi |
| 66 | + |
| 67 | +kept=0 |
| 68 | +for f in "$OBJECTS_DIR"/*; do |
| 69 | + [ -f "$f" ] || continue |
| 70 | + sha="$(sha256sum "$f" | awk '{print $1}')" |
| 71 | + if grep -qx "$sha" "$selected"; then |
| 72 | + printf '%s\t%s\n' "$sha" "$(basename "$f")" >> "$withheld" |
| 73 | + rm -f -- "$f" |
| 74 | + else |
| 75 | + kept=$((kept + 1)) |
| 76 | + fi |
| 77 | +done |
| 78 | + |
| 79 | +count="$(wc -l < "$withheld" | tr -d ' ')" |
| 80 | +if [ "$count" -eq 0 ]; then |
| 81 | + echo "error: no extracted object matched a selected sha256 — the release" >&2 |
| 82 | + echo " containers do not carry the bytes the selection recorded." >&2 |
| 83 | + exit 1 |
| 84 | +fi |
| 85 | +if [ "$kept" -eq 0 ]; then |
| 86 | + echo "error: every extracted object was withheld; the full-surface scan" >&2 |
| 87 | + echo " would cover nothing." >&2 |
| 88 | + exit 1 |
| 89 | +fi |
| 90 | + |
| 91 | +if [ -n "$MANIFEST" ]; then |
| 92 | + { |
| 93 | + echo "# cbm-virustotal-withheld-v1" |
| 94 | + echo "# reason=already-scanned-as-candidate" |
| 95 | + echo "# evidence=virustotal-candidate-results.tsv" |
| 96 | + printf 'sha256\tobject\n' |
| 97 | + cat "$withheld" |
| 98 | + } > "$MANIFEST" |
| 99 | +fi |
| 100 | + |
| 101 | +echo "withheld $count already-scanned selected executable(s); $kept object(s) remain for the surface scan" |
0 commit comments