Skip to content

Commit 175bc4b

Browse files
committed
[misc] QAT-finetune as a standalone Modal app (matches *_ab.py pattern)
Replace the launch_l40s_job.py bash wrappers with fastvideo/tests/modal/ qat_finetune_modal.py — a `modal run` app (ghcr image, hf-model-weights volume, huggingface-token secret, clone-in-container) matching the repo's other Modal launchers. Pilot vs full = --max-steps (300 vs 2000). Calls the parameterized run_qat_finetune.sh training core (kept).
1 parent 6a5356f commit 175bc4b

3 files changed

Lines changed: 110 additions & 48 deletions

File tree

examples/training/finetune/wan_t2v_1.3B/mixkit/modal_qat_full.sh

Lines changed: 0 additions & 25 deletions
This file was deleted.

examples/training/finetune/wan_t2v_1.3B/mixkit/modal_qat_pilot.sh

Lines changed: 0 additions & 23 deletions
This file was deleted.
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
"""Stage-1 QAT finetune (full-step, pre-DMD) on Modal — builds a full-step QAT
2+
Wan2.1-T2V-1.3B checkpoint that tolerates FP4 attention.
3+
4+
Runs the QAD recipe's quantization-aware finetune (the same flags as
5+
``examples/training/finetune/wan_t2v_1.3B/mixkit/finetune_qat.sh``, driven through
6+
``run_qat_finetune.sh``) on 4x L40S. Output = a *full-step* checkpoint (the
7+
distill-schedule threshold stays above ``--max-steps`` so the few-step DMD phase
8+
never triggers) — the thing the public 3-step QAD model can't give us at full
9+
sharpness.
10+
11+
Pilot vs full is just ``--max-steps`` (300 vs 2000). Run the PILOT first to shake
12+
out ATTN_QAT_TRAIN backend selection / NCCL / data download / first-validation
13+
before committing to the full run.
14+
15+
Usage (from the FastVideo repo root):
16+
17+
# PILOT — ~300 steps, watch it live (~30-45 min, a few $)
18+
modal run fastvideo/tests/modal/qat_finetune_modal.py \
19+
--max-steps 300 --validation-steps 100 --ckpt-steps 300
20+
21+
# FULL — 2000 steps, detached (survives your client exiting; ~4-6 h)
22+
modal run --detach fastvideo/tests/modal/qat_finetune_modal.py
23+
24+
Checkpoints land on the ``hf-model-weights`` volume under
25+
``/root/data/checkpoints/wan_t2v_qat_finetune`` (every ``--ckpt-steps``).
26+
Requires the Modal Secret ``huggingface-token`` (key ``HF_TOKEN``).
27+
"""
28+
import os
29+
30+
import modal
31+
32+
app = modal.App("qat-finetune")
33+
34+
model_vol = modal.Volume.from_name("hf-model-weights")
35+
hf_secret = modal.Secret.from_name("huggingface-token")
36+
image_tag = f"ghcr.io/hao-ai-lab/fastvideo/fastvideo-dev:{os.getenv('IMAGE_VERSION', 'py3.12-latest')}"
37+
38+
image = (modal.Image.from_registry(image_tag, add_python="3.12").apt_install(
39+
"cmake", "pkg-config", "build-essential", "curl", "libssl-dev", "ffmpeg", "libgl1", "libglib2.0-0",
40+
).run_commands(
41+
"curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable",
42+
"echo 'source ~/.cargo/env' >> ~/.bashrc",
43+
).env({"PATH": "/root/.cargo/bin:$PATH"}))
44+
45+
46+
def _workspace_and_train_command(git_repo: str, git_ref: str, num_gpus: int, max_steps: int,
47+
validation_steps: int, ckpt_steps: int) -> str:
48+
import shlex
49+
return f"""
50+
set -euxo pipefail
51+
source $HOME/.local/bin/env
52+
source $HOME/.cargo/env
53+
source /opt/venv/bin/activate
54+
if [ -d /FastVideo/.git ]; then
55+
cd /FastVideo && git remote set-url origin {shlex.quote(git_repo)} && git fetch --prune origin
56+
else
57+
git clone {shlex.quote(git_repo)} /FastVideo && cd /FastVideo
58+
fi
59+
git checkout {shlex.quote(git_ref)}
60+
git submodule update --init --recursive
61+
uv pip install -e ".[test]"
62+
cd fastvideo-kernel && ./build.sh && cd ..
63+
export HF_HOME=/root/data/.cache
64+
hf auth login --token "$HF_TOKEN"
65+
export NUM_GPUS={num_gpus} MAX_STEPS={max_steps} VALIDATION_STEPS={validation_steps} CKPT_STEPS={ckpt_steps}
66+
export WANDB_MODE=offline
67+
bash examples/training/finetune/wan_t2v_1.3B/mixkit/run_qat_finetune.sh
68+
"""
69+
70+
71+
@app.function(image=image, timeout=86400, volumes={"/root/data": model_vol}, secrets=[hf_secret], gpu="L40S:4")
72+
def run_qat(*, git_repo: str, git_ref: str, num_gpus: int, max_steps: int, validation_steps: int,
73+
ckpt_steps: int) -> dict:
74+
import subprocess
75+
76+
if "HF_TOKEN" not in os.environ:
77+
raise RuntimeError("HF_TOKEN not set — Modal Secret 'huggingface-token' missing the HF_TOKEN key.")
78+
cmd = _workspace_and_train_command(git_repo, git_ref, num_gpus, max_steps, validation_steps, ckpt_steps)
79+
try:
80+
subprocess.run(["/bin/bash", "-lc", cmd], env=os.environ.copy(), check=True)
81+
finally:
82+
# Persist whatever checkpoints landed (final commit; see module docstring).
83+
model_vol.commit()
84+
return {"max_steps": max_steps, "output_dir": "/root/data/checkpoints/wan_t2v_qat_finetune"}
85+
86+
87+
@app.local_entrypoint()
88+
def main(gpu: str = "L40S", num_gpus: int = 4, git_repo: str = "", git_ref: str = "spark/qad-fp4-quality",
89+
max_steps: int = 2000, validation_steps: int = 200, ckpt_steps: int = 500):
90+
"""Drive the QAT finetune from your laptop. Pilot: ``--max-steps 300
91+
--validation-steps 100 --ckpt-steps 300``. ``git_repo`` defaults to the
92+
``fork`` remote."""
93+
import subprocess
94+
95+
if not git_repo:
96+
for remote in ("fork", "origin"):
97+
try:
98+
git_repo = subprocess.check_output(["git", "config", "--get", f"remote.{remote}.url"],
99+
text=True, stderr=subprocess.DEVNULL).strip()
100+
break
101+
except subprocess.CalledProcessError:
102+
continue
103+
if not git_repo:
104+
raise RuntimeError("Could not resolve git_repo. Pass --git-repo or configure a 'fork'/'origin' remote.")
105+
106+
print(f"GPU: {gpu}:{num_gpus} ref: {git_ref} max_steps: {max_steps} "
107+
f"validation_steps: {validation_steps} ckpt_steps: {ckpt_steps} repo: {git_repo}")
108+
run_qat.with_options(gpu=f"{gpu}:{num_gpus}").remote(
109+
git_repo=git_repo, git_ref=git_ref, num_gpus=num_gpus, max_steps=max_steps,
110+
validation_steps=validation_steps, ckpt_steps=ckpt_steps)

0 commit comments

Comments
 (0)