|
| 1 | +"""Stage-1 QAT finetune (full-step, pre-DMD) on Modal — builds a full-step QAT |
| 2 | +Wan2.1-T2V-1.3B checkpoint that tolerates FP4 attention. |
| 3 | +
|
| 4 | +Runs the QAD recipe's quantization-aware finetune (the same flags as |
| 5 | +``examples/training/finetune/wan_t2v_1.3B/mixkit/finetune_qat.sh``, driven through |
| 6 | +``run_qat_finetune.sh``) on 4x L40S. Output = a *full-step* checkpoint (the |
| 7 | +distill-schedule threshold stays above ``--max-steps`` so the few-step DMD phase |
| 8 | +never triggers) — the thing the public 3-step QAD model can't give us at full |
| 9 | +sharpness. |
| 10 | +
|
| 11 | +Pilot vs full is just ``--max-steps`` (300 vs 2000). Run the PILOT first to shake |
| 12 | +out ATTN_QAT_TRAIN backend selection / NCCL / data download / first-validation |
| 13 | +before committing to the full run. |
| 14 | +
|
| 15 | +Usage (from the FastVideo repo root): |
| 16 | +
|
| 17 | + # PILOT — ~300 steps, watch it live (~30-45 min, a few $) |
| 18 | + modal run fastvideo/tests/modal/qat_finetune_modal.py \ |
| 19 | + --max-steps 300 --validation-steps 100 --ckpt-steps 300 |
| 20 | +
|
| 21 | + # FULL — 2000 steps, detached (survives your client exiting; ~4-6 h) |
| 22 | + modal run --detach fastvideo/tests/modal/qat_finetune_modal.py |
| 23 | +
|
| 24 | +Checkpoints land on the ``hf-model-weights`` volume under |
| 25 | +``/root/data/checkpoints/wan_t2v_qat_finetune`` (every ``--ckpt-steps``). |
| 26 | +Requires the Modal Secret ``huggingface-token`` (key ``HF_TOKEN``). |
| 27 | +""" |
| 28 | +import os |
| 29 | + |
| 30 | +import modal |
| 31 | + |
| 32 | +app = modal.App("qat-finetune") |
| 33 | + |
| 34 | +model_vol = modal.Volume.from_name("hf-model-weights") |
| 35 | +hf_secret = modal.Secret.from_name("huggingface-token") |
| 36 | +image_tag = f"ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev:{os.getenv('IMAGE_VERSION', 'py3.12-latest')}" |
| 37 | + |
| 38 | +image = (modal.Image.from_registry(image_tag, add_python="3.12").apt_install( |
| 39 | + "cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg", "libgl1", "libglib2.0-0", |
| 40 | +).run_commands( |
| 41 | + "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable", |
| 42 | + "echo 'source ~/.cargo/env' >> ~/.bashrc", |
| 43 | +).env({"PATH": "/root/.cargo/bin:$PATH"})) |
| 44 | + |
| 45 | + |
| 46 | +def _workspace_and_train_command(git_repo: str, git_ref: str, num_gpus: int, max_steps: int, |
| 47 | + validation_steps: int, ckpt_steps: int) -> str: |
| 48 | + import shlex |
| 49 | + return f""" |
| 50 | +set -euxo pipefail |
| 51 | +source $HOME/.local/bin/env |
| 52 | +source $HOME/.cargo/env |
| 53 | +source /opt/venv/bin/activate |
| 54 | +if [ -d /FastVideo/.git ]; then |
| 55 | + cd /FastVideo && git remote set-url origin {shlex.quote(git_repo)} && git fetch --prune origin |
| 56 | +else |
| 57 | + git clone {shlex.quote(git_repo)} /FastVideo && cd /FastVideo |
| 58 | +fi |
| 59 | +git checkout {shlex.quote(git_ref)} |
| 60 | +git submodule update --init --recursive |
| 61 | +uv pip install -e ".[test]" |
| 62 | +cd fastvideo-kernel && ./build.sh && cd .. |
| 63 | +export HF_HOME=/root/data/.cache |
| 64 | +hf auth login --token "$HF_TOKEN" |
| 65 | +export NUM_GPUS={num_gpus} MAX_STEPS={max_steps} VALIDATION_STEPS={validation_steps} CKPT_STEPS={ckpt_steps} |
| 66 | +export WANDB_MODE=offline |
| 67 | +bash examples/training/finetune/wan_t2v_1.3B/mixkit/run_qat_finetune.sh |
| 68 | +""" |
| 69 | + |
| 70 | + |
| 71 | +@app.function(image=image, timeout=86400, volumes={"/root/data": model_vol}, secrets=[hf_secret], gpu="L40S:4") |
| 72 | +def run_qat(*, git_repo: str, git_ref: str, num_gpus: int, max_steps: int, validation_steps: int, |
| 73 | + ckpt_steps: int) -> dict: |
| 74 | + import subprocess |
| 75 | + |
| 76 | + if "HF_TOKEN" not in os.environ: |
| 77 | + raise RuntimeError("HF_TOKEN not set — Modal Secret 'huggingface-token' missing the HF_TOKEN key.") |
| 78 | + cmd = _workspace_and_train_command(git_repo, git_ref, num_gpus, max_steps, validation_steps, ckpt_steps) |
| 79 | + try: |
| 80 | + subprocess.run(["/bin/bash", "-lc", cmd], env=os.environ.copy(), check=True) |
| 81 | + finally: |
| 82 | + # Persist whatever checkpoints landed (final commit; see module docstring). |
| 83 | + model_vol.commit() |
| 84 | + return {"max_steps": max_steps, "output_dir": "/root/data/checkpoints/wan_t2v_qat_finetune"} |
| 85 | + |
| 86 | + |
| 87 | +@app.local_entrypoint() |
| 88 | +def main(gpu: str = "L40S", num_gpus: int = 4, git_repo: str = "", git_ref: str = "spark/qad-fp4-quality", |
| 89 | + max_steps: int = 2000, validation_steps: int = 200, ckpt_steps: int = 500): |
| 90 | + """Drive the QAT finetune from your laptop. Pilot: ``--max-steps 300 |
| 91 | + --validation-steps 100 --ckpt-steps 300``. ``git_repo`` defaults to the |
| 92 | + ``fork`` remote.""" |
| 93 | + import subprocess |
| 94 | + |
| 95 | + if not git_repo: |
| 96 | + for remote in ("fork", "origin"): |
| 97 | + try: |
| 98 | + git_repo = subprocess.check_output(["git", "config", "--get", f"remote.{remote}.url"], |
| 99 | + text=True, stderr=subprocess.DEVNULL).strip() |
| 100 | + break |
| 101 | + except subprocess.CalledProcessError: |
| 102 | + continue |
| 103 | + if not git_repo: |
| 104 | + raise RuntimeError("Could not resolve git_repo. Pass --git-repo or configure a 'fork'/'origin' remote.") |
| 105 | + |
| 106 | + print(f"GPU: {gpu}:{num_gpus} ref: {git_ref} max_steps: {max_steps} " |
| 107 | + f"validation_steps: {validation_steps} ckpt_steps: {ckpt_steps} repo: {git_repo}") |
| 108 | + run_qat.with_options(gpu=f"{gpu}:{num_gpus}").remote( |
| 109 | + git_repo=git_repo, git_ref=git_ref, num_gpus=num_gpus, max_steps=max_steps, |
| 110 | + validation_steps=validation_steps, ckpt_steps=ckpt_steps) |
0 commit comments