[ci][diagnostic]: capture GameCraft FA4 candidate on GB200

Author: SolitaryThinker

.buildkite/scripts/gamecraft_candidate_log.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Emit and recover a small GameCraft MP4 through a Buildkite job log."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import binascii
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+
+MARKER = "FV_GAMECRAFT_FA4_MP4"
+CHUNK_BYTES = 3 * 1024
+MAX_MEDIA_BYTES = 1024 * 1024
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def emit(media_path: Path) -> None:
+    if media_path.suffix.lower() != ".mp4":
+        raise ValueError(f"Candidate must be an MP4: {media_path}")
+
+    data = media_path.read_bytes()
+    size = len(data)
+    if not 0 < size <= MAX_MEDIA_BYTES:
+        raise ValueError(f"Candidate size must be between 1 and {MAX_MEDIA_BYTES} bytes; got {size}")
+
+    digest = _sha256(data)
+    chunks = [data[offset:offset + CHUNK_BYTES] for offset in range(0, size, CHUNK_BYTES)]
+    print(
+        f"{MARKER}_BEGIN version=1 sha256={digest} size={size} "
+        f"chunks={len(chunks)} chunk_bytes={CHUNK_BYTES}",
+        flush=True,
+    )
+    for index, chunk in enumerate(chunks):
+        encoded = base64.b64encode(chunk).decode("ascii")
+        print(f"{MARKER}_CHUNK index={index:06d} data={encoded}", flush=True)
+    print(
+        f"{MARKER}_END version=1 sha256={digest} size={size} chunks={len(chunks)}",
+        flush=True,
+    )
+
+
+def _buildkite_output(text: str) -> str:
+    """Unwrap the public Buildkite job-log JSON response when supplied."""
+    try:
+        payload = json.loads(text)
+    except json.JSONDecodeError:
+        return text
+    if isinstance(payload, dict) and isinstance(payload.get("output"), str):
+        return payload["output"]
+    return text
+
+
+def decode(log_text: str) -> tuple[bytes, str]:
+    log_text = _buildkite_output(log_text)
+    begin_pattern = re.compile(
+        rf"{MARKER}_BEGIN version=1 sha256=([0-9a-f]{{64}}) size=([0-9]+) "
+        rf"chunks=([0-9]+) chunk_bytes=([0-9]+)"
+    )
+    end_pattern = re.compile(
+        rf"{MARKER}_END version=1 sha256=([0-9a-f]{{64}}) size=([0-9]+) chunks=([0-9]+)"
+    )
+    chunk_pattern = re.compile(rf"{MARKER}_CHUNK index=([0-9]{{6}}) data=([A-Za-z0-9+/]+={{0,2}})")
+
+    begin_matches = begin_pattern.findall(log_text)
+    end_matches = end_pattern.findall(log_text)
+    if len(begin_matches) != 1 or len(end_matches) != 1:
+        raise ValueError(
+            "Expected exactly one candidate envelope; "
+            f"found {len(begin_matches)} begin and {len(end_matches)} end markers"
+        )
+
+    digest, size_text, count_text, chunk_bytes_text = begin_matches[0]
+    end_digest, end_size_text, end_count_text = end_matches[0]
+    if (digest, size_text, count_text) != (end_digest, end_size_text, end_count_text):
+        raise ValueError("Candidate begin/end metadata does not match")
+
+    size = int(size_text)
+    count = int(count_text)
+    chunk_bytes = int(chunk_bytes_text)
+    if not 0 < size <= MAX_MEDIA_BYTES:
+        raise ValueError(f"Candidate size is outside the accepted range: {size}")
+    if chunk_bytes != CHUNK_BYTES:
+        raise ValueError(f"Unexpected chunk size: {chunk_bytes}")
+
+    matches = chunk_pattern.findall(log_text)
+    if len(matches) != count:
+        raise ValueError(f"Expected {count} candidate chunks; found {len(matches)}")
+
+    encoded_chunks: dict[int, str] = {}
+    for index_text, encoded in matches:
+        index = int(index_text)
+        if index in encoded_chunks:
+            raise ValueError(f"Duplicate candidate chunk: {index}")
+        encoded_chunks[index] = encoded
+    if sorted(encoded_chunks) != list(range(count)):
+        raise ValueError("Candidate chunk sequence is incomplete or out of range")
+
+    try:
+        data = b"".join(base64.b64decode(encoded_chunks[index], validate=True) for index in range(count))
+    except binascii.Error as error:
+        raise ValueError(f"Candidate chunk is not valid base64: {error}") from error
+    if len(data) != size:
+        raise ValueError(f"Decoded candidate size mismatch: expected {size}, got {len(data)}")
+    actual_digest = _sha256(data)
+    if actual_digest != digest:
+        raise ValueError(f"Decoded candidate sha256 mismatch: expected {digest}, got {actual_digest}")
+    return data, digest
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    emit_parser = subparsers.add_parser("emit", help="Encode one MP4 to stdout between strict log markers")
+    emit_parser.add_argument("media_path", type=Path)
+
+    decode_parser = subparsers.add_parser("decode", help="Recover and verify one MP4 from a raw or public JSON log")
+    decode_parser.add_argument("--input", type=Path, help="Log file (default: stdin)")
+    decode_parser.add_argument("--output", type=Path, required=True)
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = _parse_args()
+    if args.command == "emit":
+        emit(args.media_path)
+        return 0
+
+    log_text = args.input.read_text() if args.input else sys.stdin.read()
+    data, digest = decode(log_text)
+    if args.output.exists():
+        raise FileExistsError(f"Refusing to overwrite existing output: {args.output}")
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_bytes(data)
+    print(f"Recovered {len(data)} bytes with sha256={digest} to {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/scripts/lanes/ssim.sh

@@ -25,6 +25,19 @@ if [ "$selected" != all ]; then
   done
 fi
 
+# This disposable branch defaults to a focused GameCraft FA4 candidate run.
+# The canonical lane remains available for local inspection by setting this to
+# zero. Always exit 2 in candidate mode so the run cannot be mistaken for a
+# passing quality gate and cannot trigger the production exit-1 retry policy.
+candidate_enabled=${FASTVIDEO_GAMECRAFT_FA4_CANDIDATE_LOG:-1}
+force_candidate_exit() {
+  trap - EXIT
+  exit 2
+}
+if [ "$candidate_enabled" = 1 ]; then
+  trap force_candidate_exit EXIT
+fi
+
 # MoGe's utils3d dependency builds glcontext from source on ARM64. The current
 # runner image predates the baked-in X11 headers below, so keep this guarded
 # bootstrap until every deployed image digest contains libx11-dev.
@@ -37,4 +50,156 @@ fi
 uv pip install git+https://github.com/microsoft/MoGe.git
 uv pip install k_diffusion einops_exts alias_free_torch torchsde
 
+if [ "$candidate_enabled" = 1 ]; then
+  checkout_sha=$(git rev-parse HEAD)
+  echo "+++ GameCraft T2V current-FA4 candidate on GB200"
+  echo "checkout_sha=$checkout_sha"
+  echo "buildkite_commit=${BUILDKITE_COMMIT:-<unset>}"
+  if [ -n "${BUILDKITE_COMMIT:-}" ] && [ "$checkout_sha" != "$BUILDKITE_COMMIT" ]; then
+    echo "Checkout does not match BUILDKITE_COMMIT" >&2
+    exit 2
+  fi
+
+  visible_gpus=${CUDA_VISIBLE_DEVICES:-0}
+  candidate_gpu=${visible_gpus%%,*}
+  candidate_log=/tmp/fastvideo-gamecraft-fa4-candidate-pytest.log
+  candidate_sentinel=$(mktemp /tmp/fastvideo-gamecraft-fa4-candidate.XXXXXX)
+  candidate_generated_root=fastvideo/tests/ssim/generated_videos/default
+  candidate_env=(
+    "CUDA_VISIBLE_DEVICES=$candidate_gpu"
+    "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False"
+    "FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN"
+    "FASTVIDEO_FA4=1"
+    "FASTVIDEO_SSIM_BOOTSTRAP_MODE=0"
+    "FASTVIDEO_SSIM_FULL_QUALITY=0"
+  )
+
+  if env "${candidate_env[@]}" python - <<'PY'
+import os
+
+import torch
+
+print(f"requested_backend={os.environ['FASTVIDEO_ATTENTION_BACKEND']}")
+print(f"requested_FASTVIDEO_FA4={os.environ['FASTVIDEO_FA4']}")
+print(f"torch_version={torch.__version__}")
+print(f"torch_cuda_version={torch.version.cuda}")
+print(f"cuda_available={torch.cuda.is_available()}")
+print(f"cuda_visible_devices={os.environ.get('CUDA_VISIBLE_DEVICES', '<unset>')}")
+print(f"gpu_count={torch.cuda.device_count()}")
+
+if torch.cuda.device_count() != 1:
+    raise RuntimeError(f"Expected exactly one visible candidate GPU, got {torch.cuda.device_count()}")
+gpu_name = torch.cuda.get_device_name(0)
+print(f"gpu[0]={gpu_name} capability={torch.cuda.get_device_capability(0)}")
+if "GB200" not in gpu_name:
+    raise RuntimeError(f"Candidate must run on GB200, got {gpu_name}")
+
+from fastvideo.attention.selector import get_attn_backend
+from fastvideo.attention.utils.flash_attn_default import fa_version
+from fastvideo.platforms import AttentionBackendEnum
+
+requested_backend = AttentionBackendEnum[os.environ["FASTVIDEO_ATTENTION_BACKEND"]]
+resolved_backend = get_attn_backend(
+    128,
+    torch.bfloat16,
+    supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN,),
+    requested=requested_backend,
+)
+print(f"resolved_backend={resolved_backend.get_name()}")
+print(f"resolved_flash_attention=FA{fa_version}")
+if fa_version != "4":
+    raise RuntimeError(f"Candidate must use FA4, resolved FA{fa_version}")
+PY
+  then
+    probe_rc=0
+  else
+    probe_rc=$?
+  fi
+  if [ "$probe_rc" -ne 0 ]; then
+    echo "GameCraft candidate environment probe failed with rc=$probe_rc" >&2
+    exit 2
+  fi
+
+  if env "${candidate_env[@]}" python -m pytest \
+      fastvideo/tests/ssim/test_gamecraft_similarity.py::test_gamecraft_t2v_similarity \
+      -vs >"$candidate_log" 2>&1
+  then
+    test_rc=0
+  else
+    test_rc=$?
+  fi
+
+  echo "+++ GameCraft candidate pytest tail"
+  python - "$candidate_log" <<'PY'
+import sys
+from pathlib import Path
+
+log_path = Path(sys.argv[1])
+data = log_path.read_bytes()
+print(f"pytest_log_bytes={len(data)}")
+tail = data[-131072:].decode("utf-8", errors="replace").replace("\r", "\n")
+lines = tail.splitlines()[-160:]
+for line in lines:
+    print(line[:2000])
+PY
+  echo "pytest_rc=$test_rc"
+
+  candidate_videos=()
+  if [ -d "$candidate_generated_root" ]; then
+    mapfile -d '' -t candidate_videos < <(
+      find "$candidate_generated_root" -type f -name '*.mp4' -newer "$candidate_sentinel" -print0
+    )
+  fi
+  if [ "${#candidate_videos[@]}" -ne 1 ]; then
+    echo "Expected exactly one newly generated candidate MP4; found ${#candidate_videos[@]}" >&2
+    printf 'candidate_path=%s\n' "${candidate_videos[@]}" >&2
+    exit 2
+  fi
+  candidate_video=${candidate_videos[0]}
+  case "$candidate_video" in
+    */default/GB200_reference_videos/HunyuanGameCraft-T2V/FLASH_ATTN/*.mp4) ;;
+    *)
+      echo "Candidate path is outside the expected GB200 T2V subtree: $candidate_video" >&2
+      exit 2
+      ;;
+  esac
+
+  candidate_results=()
+  mapfile -d '' -t candidate_results < <(
+    find "$(dirname "$candidate_video")" -type f -name '*_ssim.json' -newer "$candidate_sentinel" -print0
+  )
+  if [ "${#candidate_results[@]}" -ne 1 ]; then
+    echo "Expected exactly one newly generated SSIM JSON; found ${#candidate_results[@]}" >&2
+    exit 2
+  fi
+  candidate_result=${candidate_results[0]}
+
+  python - "$candidate_video" "$candidate_result" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+video_path = Path(sys.argv[1]).resolve()
+result_path = Path(sys.argv[2])
+result = json.loads(result_path.read_text())
+if Path(result["generated_video"]).resolve() != video_path:
+    raise RuntimeError("SSIM JSON does not describe the candidate MP4")
+if result["parameters"]["num_inference_steps"] != 20:
+    raise RuntimeError("Candidate did not use the expected 20 inference steps")
+print(f"candidate_path={video_path}")
+print(f"candidate_ssim_mean={result['mean_ssim']}")
+print(f"candidate_ssim_min={result['min_ssim']}")
+print(f"candidate_ssim_max={result['max_ssim']}")
+print(f"candidate_prompt={result['parameters']['prompt']}")
+PY
+
+  echo "FV_GAMECRAFT_FA4_SSIM_JSON_BEGIN"
+  cat "$candidate_result"
+  echo "FV_GAMECRAFT_FA4_SSIM_JSON_END"
+  echo "+++ GameCraft candidate MP4 log envelope"
+  python .buildkite/scripts/gamecraft_candidate_log.py emit "$candidate_video"
+  echo "--- GameCraft candidate complete: probe_rc=$probe_rc pytest_rc=$test_rc (forced lane rc=2)"
+  exit 2
+fi
+
 exec python fastvideo/tests/ssim/ci_runner.py "${args[@]}"