[ci]: add SSIM reference bootstrap flow (#1522) (#1547)

Author: AtharvRN

.buildkite/scripts/pr_test.sh

@@ -80,6 +80,23 @@ MODAL_ENV="BUILDKITE_REPO=$BUILDKITE_REPO BUILDKITE_COMMIT=$BUILDKITE_COMMIT BUI
 
 POST_RUN_HOOK=""
 
+is_truthy() {
+    case "${1:-}" in
+        1|true|TRUE|yes|YES|on|ON) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+ssim_bootstrap_args() {
+    local title="${PR_TITLE:-}"
+    local message="${BUILDKITE_MESSAGE:-}"
+    if is_truthy "${FASTVIDEO_SSIM_BOOTSTRAP_MODE:-}" \
+        || [[ "$title" == *"[new-model]"* ]] \
+        || [[ "$message" == *"[new-model]"* ]]; then
+        printf ' --bootstrap-mode'
+    fi
+}
+
 upload_performance_artifacts() {
     SHORT_SHA=${BUILDKITE_COMMIT:0:7}
     LOCAL_DIR="downloaded_reports"
@@ -172,7 +189,12 @@ case "$TEST_TYPE" in
         ;;
     "ssim")
         log "Running SSIM tests..."
-        MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run $MODAL_SSIM_TEST_FILE::run_ssim_tests"
+        SSIM_BOOTSTRAP_ARGS=$(ssim_bootstrap_args)
+        if [ -n "$SSIM_BOOTSTRAP_ARGS" ]; then
+            log "SSIM bootstrap mode enabled for new-model reference draft generation"
+        fi
+        MODAL_COMMAND="$MODAL_ENV HF_API_KEY=$HF_API_KEY python3 -m modal run "
+        MODAL_COMMAND+="$MODAL_SSIM_TEST_FILE::run_ssim_tests$SSIM_BOOTSTRAP_ARGS"
         ;;
     "training")
         log "Running training tests..."

.github/workflows/ci-slash-commands.yml

@@ -52,6 +52,7 @@ jobs:
             core.setOutput('pr_sha', pr.head.sha);
             core.setOutput('pr_branch', pr.head.ref);
             core.setOutput('pr_number', String(prNumber));
+            core.setOutput('pr_title', pr.title);
 
       - name: Trigger Full Suite
         if: steps.perm.outputs.has_write == 'true'
@@ -60,6 +61,7 @@ jobs:
           PR_SHA: ${{ steps.label.outputs.pr_sha }}
           PR_BRANCH: ${{ steps.label.outputs.pr_branch }}
           PR_NUMBER: ${{ steps.label.outputs.pr_number }}
+          PR_TITLE: ${{ steps.label.outputs.pr_title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -71,6 +73,7 @@ jobs:
               --arg commit "$PR_SHA" \
               --arg branch "$PR_BRANCH" \
               --arg message "Full Suite for PR #${PR_NUMBER} (via /merge)" \
+              --arg pr_title "$PR_TITLE" \
               --argjson pr_id "$PR_NUMBER" \
               '{
                 commit: $commit,
@@ -80,11 +83,12 @@ jobs:
                 pull_request_id: $pr_id,
                 pull_request_base_branch: "main",
                 env: {
-                  TEST_SCOPE: "full",
-                  FULL_SUITE: "true",
-                  PR_NUMBER: ($pr_id | tostring)
-                }
-              }')"
+                    TEST_SCOPE: "full",
+                    FULL_SUITE: "true",
+                    PR_NUMBER: ($pr_id | tostring),
+                    PR_TITLE: $pr_title
+                  }
+                }')"
 
   parse-command:
     if: >-
@@ -241,6 +245,7 @@ jobs:
           TEST_SCOPE: ${{ needs.parse-command.outputs.test_scope }}
           FULL_SUITE: ${{ needs.parse-command.outputs.full_suite }}
           TEST_TYPE: ${{ needs.parse-command.outputs.test_type }}
+          PR_TITLE: ${{ github.event.issue.title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -257,6 +262,7 @@ jobs:
               --arg full_suite "$FULL_SUITE" \
               --arg test_type "$TEST_TYPE" \
               --arg pr_number "$PR_NUMBER" \
+              --arg pr_title "$PR_TITLE" \
               '{
                 commit: $commit,
                 branch: $branch,
@@ -266,8 +272,9 @@ jobs:
                 pull_request_base_branch: "main",
                 env: {
                   TEST_SCOPE: $test_scope,
-                  FULL_SUITE: $full_suite,
-                  TEST_TYPE: $test_type,
-                  PR_NUMBER: $pr_number
-                }
-              }')"
+                    FULL_SUITE: $full_suite,
+                    TEST_TYPE: $test_type,
+                    PR_NUMBER: $pr_number,
+                    PR_TITLE: $pr_title
+                  }
+                }')"

.github/workflows/ci-trigger-full-suite.yml

@@ -56,6 +56,7 @@ jobs:
           PR_SHA: ${{ github.event.pull_request.head.sha }}
           PR_BRANCH: ${{ github.event.pull_request.head.ref }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          PR_TITLE: ${{ github.event.pull_request.title }}
           BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
           BK_PIPELINE: ${{ vars.BUILDKITE_PIPELINE_SLUG }}
         run: |
@@ -67,6 +68,7 @@ jobs:
               --arg commit "$PR_SHA" \
               --arg branch "$PR_BRANCH" \
               --arg message "Full Suite for PR #${PR_NUMBER}" \
+              --arg pr_title "$PR_TITLE" \
               --argjson pr_id "$PR_NUMBER" \
               '{
                 commit: $commit,
@@ -78,6 +80,7 @@ jobs:
                 env: {
                   TEST_SCOPE: "full",
                   FULL_SUITE: "true",
-                  PR_NUMBER: ($pr_id | tostring)
+                  PR_NUMBER: ($pr_id | tostring),
+                  PR_TITLE: $pr_title
                 }
               }')"

docs/contributing/testing.md

@@ -180,6 +180,30 @@ python fastvideo/tests/ssim/reference_videos_cli.py copy-local \
   --device-folder L40S_reference_videos
 ```
 
+### SSIM Bootstrap Mode
+
+Normal SSIM runs are strict: if a reference video or latent is missing, the
+test fails. For new-model PRs, CI can run SSIM in bootstrap mode so missing
+references are uploaded as draft artifacts for review instead of immediately
+blocking on a missing canonical reference.
+
+Buildkite enables SSIM bootstrap mode when either condition is true:
+
+- the PR title or Buildkite message contains `[new-model]`;
+- `FASTVIDEO_SSIM_BOOTSTRAP_MODE=1` is set for the Buildkite job.
+
+Bootstrap mode passes `--ssim-bootstrap-mode` to pytest. When a generated
+artifact is available, the test uploads it under the `drafts/...` namespace in
+the SSIM reference repo and marks that case as expected-failed. After reviewing
+the draft, promote it into the canonical reference layout:
+
+```bash
+python fastvideo/tests/ssim/reference_videos_cli.py promote-draft \
+  --quality-tier default \
+  --device-folder L40S_reference_videos \
+  --model-id <model_id>
+```
+
 ## CI Integration
 
 FastVideo CI tests are orchestrated by Buildkite and run on Modal GPU

fastvideo/tests/modal/ssim_test.py

@@ -550,6 +550,7 @@ def _spawn_ssim_task(
     log_dir: str,
     task_index: int,
     pytest_extra_args: list[str],
+    hf_api_key: str,
 ) -> _RunningTask:
     import shlex
 
@@ -559,6 +560,7 @@ def _spawn_ssim_task(
     command = f"set -euo pipefail && source $HOME/.local/bin/env && source /opt/venv/bin/activate && {pytest_command}"
     env = os.environ.copy()
     env["HF_HOME"] = "/root/data/.cache"
+    env["HF_API_KEY"] = hf_api_key
     # MultiprocExecutor returns CUDA tensors through mp pipes (CUDA IPC).
     # On kernels without pidfd_open support, PyTorch fails when
     # expandable_segments=True. Force False for CI compatibility.
@@ -675,6 +677,7 @@ def _build_pytest_extra_args(
     ssim_full_quality: bool,
     ssim_reference_repo: str,
     skip_ssim_reference_download: bool,
+    ssim_bootstrap_mode: bool,
     pytest_k: str,
 ) -> list[str]:
     args = []
@@ -684,6 +687,8 @@ def _build_pytest_extra_args(
         args.extend(["--ssim-reference-repo", ssim_reference_repo.strip()])
     if skip_ssim_reference_download:
         args.append("--skip-ssim-reference-download")
+    if ssim_bootstrap_mode:
+        args.append("--ssim-bootstrap-mode")
     if pytest_k.strip():
         args.extend(["-k", pytest_k.strip()])
     return args
@@ -693,6 +698,7 @@ def _schedule_ssim_tasks(
     repo_root: str,
     tasks: list[SSIMTask],
     pytest_extra_args: list[str],
+    hf_api_key: str,
     fail_fast: bool = True,
 ) -> dict[int, _TaskResult]:
     import tempfile
@@ -735,6 +741,7 @@ def _schedule_ssim_tasks(
                 log_dir=log_dir,
                 task_index=task.task_id,
                 pytest_extra_args=pytest_extra_args,
+                hf_api_key=hf_api_key,
             )
             print(f"Started {task.test_name} on GPUs {','.join(assigned_gpu_ids)}")
             running_tasks.append(running_task)
@@ -885,6 +892,7 @@ def run_ssim_partition(
     ssim_full_quality: bool = False,
     ssim_reference_repo: str = "",
     skip_ssim_reference_download: bool = False,
+    ssim_bootstrap_mode: bool = False,
     pytest_k: str = "",
     sync_generated_to_volume: bool = False,
     generated_volume_subdir: str = "",
@@ -913,12 +921,14 @@ def run_ssim_partition(
         ssim_full_quality=ssim_full_quality,
         ssim_reference_repo=ssim_reference_repo,
         skip_ssim_reference_download=skip_ssim_reference_download,
+        ssim_bootstrap_mode=ssim_bootstrap_mode,
         pytest_k=pytest_k,
     )
     results = _schedule_ssim_tasks(
         repo_root,
         partition,
         pytest_extra_args=pytest_extra_args,
+        hf_api_key=hf_api_key,
         fail_fast=fail_fast,
     )
     summaries = _collect_task_summaries(partition, results)
@@ -955,6 +965,7 @@ def run_ssim_tests(
     full_quality: bool = False,
     reference_repo: str = "",
     skip_reference_download: bool = False,
+    bootstrap_mode: bool = False,
     pytest_k: str = "",
     sync_generated_to_volume: bool = False,
     generated_volume_subdir: str = "",
@@ -975,15 +986,24 @@ def run_ssim_tests(
         print(f"Selected model ids: {model_ids}")
     if pytest_k.strip():
         print(f"Using pytest -k filter: {pytest_k}")
+    if bootstrap_mode:
+        print(
+            "SSIM bootstrap mode enabled: missing references will upload "
+            "draft artifacts and xfail."
+        )
     quality_tier = _resolve_output_quality_tier(full_quality)
     if sync_generated_to_volume:
         resolved_subdir = _resolve_generated_volume_subdir(
             generated_volume_subdir,
             resolved_git_commit,
         )
+        generated_volume_path = _build_generated_volume_relative_path(
+            generated_volume_subdir=resolved_subdir,
+            quality_tier=quality_tier,
+        )
         print(
             "Raw generated videos will be saved to Modal volume path: "
-            f"{_build_generated_volume_relative_path(generated_volume_subdir=resolved_subdir, quality_tier=quality_tier)}"
+            f"{generated_volume_path}"
         )
     else:
         resolved_subdir = ""
@@ -998,6 +1018,7 @@ def run_ssim_tests(
         ssim_full_quality=full_quality,
         ssim_reference_repo=reference_repo,
         skip_ssim_reference_download=skip_reference_download,
+        ssim_bootstrap_mode=bootstrap_mode,
         pytest_k=pytest_k,
         sync_generated_to_volume=sync_generated_to_volume,
         generated_volume_subdir=resolved_subdir,

fastvideo/tests/modal/test_ssim_test.py

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+
+class _FakeImage:
+
+    @classmethod
+    def from_registry(cls, *_args, **_kwargs):
+        return cls()
+
+    def apt_install(self, *_args, **_kwargs):
+        return self
+
+    def run_commands(self, *_args, **_kwargs):
+        return self
+
+    def env(self, *_args, **_kwargs):
+        return self
+
+
+class _FakeVolume:
+
+    @classmethod
+    def from_name(cls, *_args, **_kwargs):
+        return cls()
+
+
+class _FakeApp:
+
+    def function(self, *_args, **_kwargs):
+        def decorator(func):
+            return func
+
+        return decorator
+
+    def local_entrypoint(self, *_args, **_kwargs):
+        def decorator(func):
+            return func
+
+        return decorator
+
+
+def _load_ssim_test_module(monkeypatch):
+    fake_modal = types.SimpleNamespace(
+        App=lambda: _FakeApp(),
+        Image=_FakeImage,
+        Volume=_FakeVolume,
+    )
+    monkeypatch.setitem(sys.modules, "modal", fake_modal)
+    module_path = Path(__file__).with_name("ssim_test.py")
+    spec = importlib.util.spec_from_file_location("modal_ssim_test_under_test", module_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_spawn_ssim_task_passes_hf_api_key_to_pytest_env(monkeypatch, tmp_path):
+    module = _load_ssim_test_module(monkeypatch)
+    captured = {}
+
+    class FakeProcess:
+        pass
+
+    def fake_popen(*args, **kwargs):
+        captured["args"] = args
+        captured["kwargs"] = kwargs
+        return FakeProcess()
+
+    monkeypatch.setattr(module.subprocess, "Popen", fake_popen)
+
+    task = module.SSIMTask(
+        task_id=0,
+        test_file="fastvideo/tests/ssim/test_example.py",
+        required_gpus=1,
+        model_id="model-a",
+    )
+
+    running_task = module._spawn_ssim_task(
+        task=task,
+        repo_root=str(tmp_path),
+        assigned_gpu_ids=["2"],
+        log_dir=str(tmp_path),
+        task_index=0,
+        pytest_extra_args=["--ssim-bootstrap-mode"],
+        hf_api_key="hf_test_token",
+    )
+
+    env = captured["kwargs"]["env"]
+    assert env["HF_API_KEY"] == "hf_test_token"
+    assert env["HF_HOME"] == "/root/data/.cache"
+    assert env["CUDA_VISIBLE_DEVICES"] == "2"
+    assert env["FASTVIDEO_SSIM_MODEL_ID"] == "model-a"
+    assert running_task.process.__class__ is FakeProcess
+    running_task.log_handle.close()