From be1df9fec5be32e6177e53061d6ab2883686ac6a Mon Sep 17 00:00:00 2001 From: William Lin <8941107+SolitaryThinker@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:14:19 -0700 Subject: [PATCH] [ci][diagnostic]: capture GameCraft FA4 candidate on GB200 --- .buildkite/scripts/gamecraft_candidate_log.py | 148 ++++++++++++++++ .buildkite/scripts/lanes/ssim.sh | 165 ++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 .buildkite/scripts/gamecraft_candidate_log.py diff --git a/.buildkite/scripts/gamecraft_candidate_log.py b/.buildkite/scripts/gamecraft_candidate_log.py new file mode 100644 index 0000000000..939e71097a --- /dev/null +++ b/.buildkite/scripts/gamecraft_candidate_log.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Emit and recover a small GameCraft MP4 through a Buildkite job log.""" + +from __future__ import annotations + +import argparse +import base64 +import binascii +import hashlib +import html +import json +import re +import sys +from pathlib import Path + +MARKER = "FV_GAMECRAFT_FA4_MP4" +CHUNK_BYTES = 3 * 1024 +MAX_MEDIA_BYTES = 1024 * 1024 + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def emit(media_path: Path) -> None: + if media_path.suffix.lower() != ".mp4": + raise ValueError(f"Candidate must be an MP4: {media_path}") + + data = media_path.read_bytes() + size = len(data) + if not 0 < size <= MAX_MEDIA_BYTES: + raise ValueError(f"Candidate size must be between 1 and {MAX_MEDIA_BYTES} bytes; got {size}") + + digest = _sha256(data) + chunks = [data[offset:offset + CHUNK_BYTES] for offset in range(0, size, CHUNK_BYTES)] + print( + f"{MARKER}_BEGIN version=1 sha256={digest} size={size} " + f"chunks={len(chunks)} chunk_bytes={CHUNK_BYTES}", + flush=True, + ) + for index, chunk in enumerate(chunks): + encoded = base64.b64encode(chunk).decode("ascii") + print(f"{MARKER}_CHUNK index={index:06d} data={encoded}", flush=True) + print( + f"{MARKER}_END version=1 sha256={digest} size={size} chunks={len(chunks)}", + flush=True, + ) + + +def _buildkite_output(text: str) -> str: + """Unwrap Buildkite's public JSON and reverse its HTML entity escaping.""" + try: + payload = json.loads(text) + except json.JSONDecodeError: + return html.unescape(text) + if isinstance(payload, dict) and isinstance(payload.get("output"), str): + return html.unescape(payload["output"]) + return html.unescape(text) + + +def decode(log_text: str) -> tuple[bytes, str]: + log_text = _buildkite_output(log_text) + begin_pattern = re.compile( + rf"{MARKER}_BEGIN version=1 sha256=([0-9a-f]{{64}}) size=([0-9]+) " + rf"chunks=([0-9]+) chunk_bytes=([0-9]+)" + ) + end_pattern = re.compile( + rf"{MARKER}_END version=1 sha256=([0-9a-f]{{64}}) size=([0-9]+) chunks=([0-9]+)" + ) + chunk_pattern = re.compile(rf"{MARKER}_CHUNK index=([0-9]{{6}}) data=([A-Za-z0-9+/]+={{0,2}})") + + begin_matches = begin_pattern.findall(log_text) + end_matches = end_pattern.findall(log_text) + if len(begin_matches) != 1 or len(end_matches) != 1: + raise ValueError( + "Expected exactly one candidate envelope; " + f"found {len(begin_matches)} begin and {len(end_matches)} end markers" + ) + + digest, size_text, count_text, chunk_bytes_text = begin_matches[0] + end_digest, end_size_text, end_count_text = end_matches[0] + if (digest, size_text, count_text) != (end_digest, end_size_text, end_count_text): + raise ValueError("Candidate begin/end metadata does not match") + + size = int(size_text) + count = int(count_text) + chunk_bytes = int(chunk_bytes_text) + if not 0 < size <= MAX_MEDIA_BYTES: + raise ValueError(f"Candidate size is outside the accepted range: {size}") + if chunk_bytes != CHUNK_BYTES: + raise ValueError(f"Unexpected chunk size: {chunk_bytes}") + + matches = chunk_pattern.findall(log_text) + if len(matches) != count: + raise ValueError(f"Expected {count} candidate chunks; found {len(matches)}") + + encoded_chunks: dict[int, str] = {} + for index_text, encoded in matches: + index = int(index_text) + if index in encoded_chunks: + raise ValueError(f"Duplicate candidate chunk: {index}") + encoded_chunks[index] = encoded + if sorted(encoded_chunks) != list(range(count)): + raise ValueError("Candidate chunk sequence is incomplete or out of range") + + try: + data = b"".join(base64.b64decode(encoded_chunks[index], validate=True) for index in range(count)) + except binascii.Error as error: + raise ValueError(f"Candidate chunk is not valid base64: {error}") from error + if len(data) != size: + raise ValueError(f"Decoded candidate size mismatch: expected {size}, got {len(data)}") + actual_digest = _sha256(data) + if actual_digest != digest: + raise ValueError(f"Decoded candidate sha256 mismatch: expected {digest}, got {actual_digest}") + return data, digest + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + emit_parser = subparsers.add_parser("emit", help="Encode one MP4 to stdout between strict log markers") + emit_parser.add_argument("media_path", type=Path) + + decode_parser = subparsers.add_parser("decode", help="Recover and verify one MP4 from a raw or public JSON log") + decode_parser.add_argument("--input", type=Path, help="Log file (default: stdin)") + decode_parser.add_argument("--output", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + if args.command == "emit": + emit(args.media_path) + return 0 + + log_text = args.input.read_text() if args.input else sys.stdin.read() + data, digest = decode(log_text) + if args.output.exists(): + raise FileExistsError(f"Refusing to overwrite existing output: {args.output}") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(data) + print(f"Recovered {len(data)} bytes with sha256={digest} to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.buildkite/scripts/lanes/ssim.sh b/.buildkite/scripts/lanes/ssim.sh index 54bc0d4068..798b42f4bb 100755 --- a/.buildkite/scripts/lanes/ssim.sh +++ b/.buildkite/scripts/lanes/ssim.sh @@ -25,6 +25,19 @@ if [ "$selected" != all ]; then done fi +# This disposable branch defaults to a focused GameCraft FA4 candidate run. +# The canonical lane remains available for local inspection by setting this to +# zero. Always exit 2 in candidate mode so the run cannot be mistaken for a +# passing quality gate and cannot trigger the production exit-1 retry policy. +candidate_enabled=${FASTVIDEO_GAMECRAFT_FA4_CANDIDATE_LOG:-1} +force_candidate_exit() { + trap - EXIT + exit 2 +} +if [ "$candidate_enabled" = 1 ]; then + trap force_candidate_exit EXIT +fi + # MoGe's utils3d dependency builds glcontext from source on ARM64. The current # runner image predates the baked-in X11 headers below, so keep this guarded # bootstrap until every deployed image digest contains libx11-dev. @@ -37,4 +50,156 @@ fi uv pip install git+https://github.com/microsoft/MoGe.git uv pip install k_diffusion einops_exts alias_free_torch torchsde +if [ "$candidate_enabled" = 1 ]; then + checkout_sha=$(git rev-parse HEAD) + echo "+++ GameCraft T2V current-FA4 candidate on GB200" + echo "checkout_sha=$checkout_sha" + echo "buildkite_commit=${BUILDKITE_COMMIT:-}" + if [ -n "${BUILDKITE_COMMIT:-}" ] && [ "$checkout_sha" != "$BUILDKITE_COMMIT" ]; then + echo "Checkout does not match BUILDKITE_COMMIT" >&2 + exit 2 + fi + + visible_gpus=${CUDA_VISIBLE_DEVICES:-0} + candidate_gpu=${visible_gpus%%,*} + candidate_log=/tmp/fastvideo-gamecraft-fa4-candidate-pytest.log + candidate_sentinel=$(mktemp /tmp/fastvideo-gamecraft-fa4-candidate.XXXXXX) + candidate_generated_root=fastvideo/tests/ssim/generated_videos/default + candidate_env=( + "CUDA_VISIBLE_DEVICES=$candidate_gpu" + "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False" + "FASTVIDEO_ATTENTION_BACKEND=FLASH_ATTN" + "FASTVIDEO_FA4=1" + "FASTVIDEO_SSIM_BOOTSTRAP_MODE=0" + "FASTVIDEO_SSIM_FULL_QUALITY=0" + ) + + if env "${candidate_env[@]}" python - <<'PY' +import os + +import torch + +print(f"requested_backend={os.environ['FASTVIDEO_ATTENTION_BACKEND']}") +print(f"requested_FASTVIDEO_FA4={os.environ['FASTVIDEO_FA4']}") +print(f"torch_version={torch.__version__}") +print(f"torch_cuda_version={torch.version.cuda}") +print(f"cuda_available={torch.cuda.is_available()}") +print(f"cuda_visible_devices={os.environ.get('CUDA_VISIBLE_DEVICES', '')}") +print(f"gpu_count={torch.cuda.device_count()}") + +if torch.cuda.device_count() != 1: + raise RuntimeError(f"Expected exactly one visible candidate GPU, got {torch.cuda.device_count()}") +gpu_name = torch.cuda.get_device_name(0) +print(f"gpu[0]={gpu_name} capability={torch.cuda.get_device_capability(0)}") +if "GB200" not in gpu_name: + raise RuntimeError(f"Candidate must run on GB200, got {gpu_name}") + +from fastvideo.attention.selector import get_attn_backend +from fastvideo.attention.utils.flash_attn_default import fa_version +from fastvideo.platforms import AttentionBackendEnum + +requested_backend = AttentionBackendEnum[os.environ["FASTVIDEO_ATTENTION_BACKEND"]] +resolved_backend = get_attn_backend( + 128, + torch.bfloat16, + supported_attention_backends=(AttentionBackendEnum.FLASH_ATTN,), + requested=requested_backend, +) +print(f"resolved_backend={resolved_backend.get_name()}") +print(f"resolved_flash_attention=FA{fa_version}") +if fa_version != "4": + raise RuntimeError(f"Candidate must use FA4, resolved FA{fa_version}") +PY + then + probe_rc=0 + else + probe_rc=$? + fi + if [ "$probe_rc" -ne 0 ]; then + echo "GameCraft candidate environment probe failed with rc=$probe_rc" >&2 + exit 2 + fi + + if env "${candidate_env[@]}" python -m pytest \ + fastvideo/tests/ssim/test_gamecraft_similarity.py::test_gamecraft_t2v_similarity \ + -vs >"$candidate_log" 2>&1 + then + test_rc=0 + else + test_rc=$? + fi + + echo "+++ GameCraft candidate pytest tail" + python - "$candidate_log" <<'PY' +import sys +from pathlib import Path + +log_path = Path(sys.argv[1]) +data = log_path.read_bytes() +print(f"pytest_log_bytes={len(data)}") +tail = data[-131072:].decode("utf-8", errors="replace").replace("\r", "\n") +lines = tail.splitlines()[-160:] +for line in lines: + print(line[:2000]) +PY + echo "pytest_rc=$test_rc" + + candidate_videos=() + if [ -d "$candidate_generated_root" ]; then + mapfile -d '' -t candidate_videos < <( + find "$candidate_generated_root" -type f -name '*.mp4' -newer "$candidate_sentinel" -print0 + ) + fi + if [ "${#candidate_videos[@]}" -ne 1 ]; then + echo "Expected exactly one newly generated candidate MP4; found ${#candidate_videos[@]}" >&2 + printf 'candidate_path=%s\n' "${candidate_videos[@]}" >&2 + exit 2 + fi + candidate_video=${candidate_videos[0]} + case "$candidate_video" in + */default/GB200_reference_videos/HunyuanGameCraft-T2V/FLASH_ATTN/*.mp4) ;; + *) + echo "Candidate path is outside the expected GB200 T2V subtree: $candidate_video" >&2 + exit 2 + ;; + esac + + candidate_results=() + mapfile -d '' -t candidate_results < <( + find "$(dirname "$candidate_video")" -type f -name '*_ssim.json' -newer "$candidate_sentinel" -print0 + ) + if [ "${#candidate_results[@]}" -ne 1 ]; then + echo "Expected exactly one newly generated SSIM JSON; found ${#candidate_results[@]}" >&2 + exit 2 + fi + candidate_result=${candidate_results[0]} + + python - "$candidate_video" "$candidate_result" <<'PY' +import json +import sys +from pathlib import Path + +video_path = Path(sys.argv[1]).resolve() +result_path = Path(sys.argv[2]) +result = json.loads(result_path.read_text()) +if Path(result["generated_video"]).resolve() != video_path: + raise RuntimeError("SSIM JSON does not describe the candidate MP4") +if result["parameters"]["num_inference_steps"] != 20: + raise RuntimeError("Candidate did not use the expected 20 inference steps") +print(f"candidate_path={video_path}") +print(f"candidate_ssim_mean={result['mean_ssim']}") +print(f"candidate_ssim_min={result['min_ssim']}") +print(f"candidate_ssim_max={result['max_ssim']}") +print(f"candidate_prompt={result['parameters']['prompt']}") +PY + + echo "FV_GAMECRAFT_FA4_SSIM_JSON_BEGIN" + cat "$candidate_result" + echo "FV_GAMECRAFT_FA4_SSIM_JSON_END" + echo "+++ GameCraft candidate MP4 log envelope" + python .buildkite/scripts/gamecraft_candidate_log.py emit "$candidate_video" + echo "--- GameCraft candidate complete: probe_rc=$probe_rc pytest_rc=$test_rc (forced lane rc=2)" + exit 2 +fi + exec python fastvideo/tests/ssim/ci_runner.py "${args[@]}"