Skip to content

Commit 5ae05b0

Browse files
SolitaryThinkerSolitaryThinker
authored
[misc]: add LTX-2 fine-tuning example recipes (#1645)
1 parent 7a592ff commit 5ae05b0

4 files changed

Lines changed: 359 additions & 0 deletions

File tree

‎docs/training/train_infra.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,10 @@ switches those layers to `ATTN_QAT_INFER`.
231231
On GB200, set `callbacks.validation.attn_qat_infer: false` to keep validation on
232232
the train-time QAT backend; the inference kernel is sm120-only.
233233

234+
User-adaptable LTX-2 fine-tuning recipes (full, LoRA, and NVFP4 QAT) live in
235+
`examples/train/configs/fine_tuning/ltx2/`, alongside the other model
236+
families under `examples/train/configs/fine_tuning/`.
237+
234238
---
235239

236240
## Training Methods

‎examples/train/configs/fine_tuning/ltx2/nvfp4_qat_t2v.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
# LTX-2 T2V NVFP4 quantization-aware fine-tune (QAT).
2+
#
3+
# Same recipe as fine_tuning/ltx2/t2v.yaml, with the deployment-matched
4+
# NVFP4 quantization from the validated
5+
# examples/train/configs/overfit_ltx2_t2v_nvfp4_qat.yaml run: the curated
6+
# attention/FFN linear set used by LTX-2 NVFP4 deployment runs its
7+
# forward through real NVFP4 GEMMs (flashinfer cutlass) with a
8+
# straight-through-estimator backward into the dense fp32 master weights.
9+
# No extra parameters or buffers are added, so FSDP sharding,
10+
# checkpointing, and export are identical to dense training. Video
11+
# attention uses ATTN_QAT_TRAIN for its quantized forward and STE
12+
# backward, then ATTN_QAT_INFER during validation. Head-dim-64 audio
13+
# attention and masked text attention remain dense.
14+
#
15+
# Validation requires an sm_120 GPU with the attn_qat_infer extension.
16+
#
17+
# Prerequisites: preprocess your dataset into the trainer's latent format
18+
# (see docs/training/data_preprocess.md;
19+
# fastvideo/pipelines/preprocess/preprocess_ltx2_overfit.py is a minimal
20+
# LTX-2 reference to adapt). Data paths, step counts, and learning rate
21+
# below are placeholders you must adapt to your workload.
22+
#
23+
# Run on multi-GPU sm_120 hardware:
24+
# NUM_GPUS=4 bash examples/train/run.sh \
25+
# examples/train/configs/fine_tuning/ltx2/nvfp4_qat_t2v.yaml
26+
#
27+
# GB200 can train and validate with ATTN_QAT_TRAIN, but cannot load the
28+
# sm_120-only inference kernel. Disable only the validation-time swap:
29+
# NUM_GPUS=4 bash examples/train/run.sh \
30+
# examples/train/configs/fine_tuning/ltx2/nvfp4_qat_t2v.yaml \
31+
# --callbacks.validation.attn_qat_infer false
32+
33+
models:
34+
student:
35+
_target_: fastvideo.train.models.ltx2.LTX2Model
36+
# Validation settings below (8 steps, guidance 1.0) assume this
37+
# distilled checkpoint.
38+
init_from: FastVideo/LTX2-Distilled-Diffusers
39+
trainable: true
40+
enable_gradient_checkpointing_type: full
41+
attention_backend: ATTN_QAT_TRAIN
42+
43+
method:
44+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
45+
46+
training:
47+
distributed:
48+
# The 13B video branch needs sharding; 4-way HSDP shard matches the
49+
# validated LTX-2 runs. Scale num_gpus/hsdp_shard_dim together.
50+
num_gpus: 4
51+
sp_size: 1
52+
tp_size: 1
53+
hsdp_replicate_dim: 1
54+
hsdp_shard_dim: 4
55+
56+
data:
57+
# REQUIRED: path to your preprocessed dataset.
58+
data_path: data/your_dataset_preprocessed
59+
dataloader_num_workers: 4
60+
train_batch_size: 1
61+
# LTX2Model requires 0.0: CFG dropout would zero post-connector
62+
# embeddings, which is not the model's unconditional input.
63+
training_cfg_rate: 0.0
64+
seed: 42
65+
# Must match your preprocessed resolution/length.
66+
# num_latent_t = (num_frames - 1) / 8 + 1.
67+
num_latent_t: 11
68+
num_height: 480
69+
num_width: 832
70+
num_frames: 81
71+
72+
optimizer:
73+
# Carried from the validated LTX-2 QAT overfit run as a starting
74+
# point; tune for your dataset size and batch configuration.
75+
learning_rate: 5.0e-5
76+
betas: [0.9, 0.999]
77+
weight_decay: 0.0
78+
lr_scheduler: constant
79+
lr_warmup_steps: 0
80+
81+
loop:
82+
# Workload-dependent: set from your dataset size and target epochs.
83+
max_train_steps: 2000
84+
gradient_accumulation_steps: 1
85+
86+
checkpoint:
87+
output_dir: outputs/ltx2_t2v_nvfp4_qat_finetune
88+
# A full training-state checkpoint is ~150GB for the 13B trainable
89+
# video branch — size the interval and total limit to your storage.
90+
training_state_checkpointing_steps: 500
91+
checkpoints_total_limit: 2
92+
resume_from_checkpoint: latest
93+
94+
tracker:
95+
trackers: [wandb]
96+
project_name: fastvideo_ltx2
97+
run_name: ltx2_t2v_nvfp4_qat_finetune
98+
99+
model:
100+
# LTX2Model.predict_noise converts the DiT's x0 output to velocity,
101+
# so the default noise-minus-clean target reproduces the official
102+
# unweighted masked-MSE (mask is all-ones for plain T2V).
103+
precondition_outputs: false
104+
enable_gradient_checkpointing_type: full
105+
106+
callbacks:
107+
grad_clip:
108+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
109+
max_grad_norm: 1.0
110+
validation:
111+
_target_: fastvideo.train.callbacks.validation.ValidationCallback
112+
pipeline_target: fastvideo.pipelines.basic.ltx2.ltx2_pipeline.LTX2Pipeline
113+
# REQUIRED: prompts to sample during training.
114+
dataset_file: data/your_dataset_preprocessed/validation_prompts.json
115+
every_steps: 250
116+
# 8-step single-pass sampling matches the distilled checkpoint
117+
# (validated in the LTX-2 overfit runs).
118+
sampling_steps: [8]
119+
guidance_scale: 1.0
120+
num_frames: 81
121+
# Validation reuses the live transformer and temporarily replaces its
122+
# ATTN_QAT_TRAIN implementations with the sm120 inference kernel.
123+
# Set false on GB200 (see header).
124+
attn_qat_infer: true
125+
126+
# quant_config selects NVFP4 QAT for the same curated linear prefixes as
127+
# LTX-2 NVFP4 deployment. They fake-quantize through real FP4 GEMMs with
128+
# an STE backward. The string resolves to NVFP4QATTrainConfig at parse.
129+
pipeline:
130+
dit_config:
131+
quant_config: nvfp4_qat_train

‎examples/train/configs/fine_tuning/ltx2/t2v.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
# LTX-2 T2V full fine-tune.
2+
#
3+
# Starting point for fine-tuning LTX-2 on your own dataset with the
4+
# modular trainer. Model wiring (checkpoint, CFG rate, output
5+
# preconditioning, sharding) follows the validated
6+
# examples/train/configs/overfit_ltx2_t2v.yaml run; data paths, step
7+
# counts, and learning rate are placeholders you must adapt — the right
8+
# values depend on your dataset and are marked below.
9+
#
10+
# Prerequisites: preprocess your dataset into the trainer's latent format
11+
# (see docs/training/data_preprocess.md;
12+
# fastvideo/pipelines/preprocess/preprocess_ltx2_overfit.py is a minimal
13+
# LTX-2 reference to adapt).
14+
#
15+
# Run:
16+
# NUM_GPUS=4 bash examples/train/run.sh \
17+
# examples/train/configs/fine_tuning/ltx2/t2v.yaml
18+
19+
models:
20+
student:
21+
_target_: fastvideo.train.models.ltx2.LTX2Model
22+
# Validation settings below (8 steps, guidance 1.0) assume this
23+
# distilled checkpoint.
24+
init_from: FastVideo/LTX2-Distilled-Diffusers
25+
trainable: true
26+
enable_gradient_checkpointing_type: full
27+
28+
method:
29+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
30+
31+
training:
32+
distributed:
33+
# The 13B video branch needs sharding; 4-way HSDP shard matches the
34+
# validated LTX-2 runs. Scale num_gpus/hsdp_shard_dim together.
35+
num_gpus: 4
36+
sp_size: 1
37+
tp_size: 1
38+
hsdp_replicate_dim: 1
39+
hsdp_shard_dim: 4
40+
41+
data:
42+
# REQUIRED: path to your preprocessed dataset.
43+
data_path: data/your_dataset_preprocessed
44+
dataloader_num_workers: 4
45+
train_batch_size: 1
46+
# LTX2Model requires 0.0: CFG dropout would zero post-connector
47+
# embeddings, which is not the model's unconditional input.
48+
training_cfg_rate: 0.0
49+
seed: 42
50+
# Must match your preprocessed resolution/length.
51+
# num_latent_t = (num_frames - 1) / 8 + 1.
52+
num_latent_t: 11
53+
num_height: 480
54+
num_width: 832
55+
num_frames: 81
56+
57+
optimizer:
58+
# Carried from the validated LTX-2 overfit run as a starting point;
59+
# tune for your dataset size and batch configuration.
60+
learning_rate: 5.0e-5
61+
betas: [0.9, 0.999]
62+
weight_decay: 0.0
63+
lr_scheduler: constant
64+
lr_warmup_steps: 0
65+
66+
loop:
67+
# Workload-dependent: set from your dataset size and target epochs.
68+
max_train_steps: 2000
69+
gradient_accumulation_steps: 1
70+
71+
checkpoint:
72+
output_dir: outputs/ltx2_t2v_finetune
73+
# A full training-state checkpoint is ~150GB for the 13B trainable
74+
# video branch — size the interval and total limit to your storage.
75+
training_state_checkpointing_steps: 500
76+
checkpoints_total_limit: 2
77+
resume_from_checkpoint: latest
78+
79+
tracker:
80+
trackers: [wandb]
81+
project_name: fastvideo_ltx2
82+
run_name: ltx2_t2v_finetune
83+
84+
model:
85+
# LTX2Model.predict_noise converts the DiT's x0 output to velocity,
86+
# so the default noise-minus-clean target reproduces the official
87+
# unweighted masked-MSE (mask is all-ones for plain T2V).
88+
precondition_outputs: false
89+
enable_gradient_checkpointing_type: full
90+
91+
callbacks:
92+
grad_clip:
93+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
94+
max_grad_norm: 1.0
95+
validation:
96+
_target_: fastvideo.train.callbacks.validation.ValidationCallback
97+
pipeline_target: fastvideo.pipelines.basic.ltx2.ltx2_pipeline.LTX2Pipeline
98+
# REQUIRED: prompts to sample during training.
99+
dataset_file: data/your_dataset_preprocessed/validation_prompts.json
100+
every_steps: 250
101+
# 8-step single-pass sampling matches the distilled checkpoint
102+
# (validated in the LTX-2 overfit runs).
103+
sampling_steps: [8]
104+
guidance_scale: 1.0
105+
num_frames: 81
106+
107+
# Required so the LTX2T2VConfig pipeline config is resolved from
108+
# init_from (without a `pipeline:` key the loader falls back to a
109+
# generic PipelineConfig and the LTX-2 DiT cannot be constructed).
110+
pipeline: {}

‎examples/train/configs/fine_tuning/ltx2/t2v_lora.yaml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# LTX-2 T2V LoRA fine-tune.
2+
#
3+
# Same recipe as fine_tuning/ltx2/t2v.yaml, but only LoRA adapters on the
4+
# attention projections train — the base weights stay frozen, so
5+
# training-state checkpoints are adapter-sized instead of ~150GB.
6+
# See that file's header for prerequisites and placeholder notes.
7+
#
8+
# Run:
9+
# NUM_GPUS=4 bash examples/train/run.sh \
10+
# examples/train/configs/fine_tuning/ltx2/t2v_lora.yaml
11+
12+
models:
13+
student:
14+
_target_: fastvideo.train.models.ltx2.LTX2Model
15+
# Validation settings below (8 steps, guidance 1.0) assume this
16+
# distilled checkpoint.
17+
init_from: FastVideo/LTX2-Distilled-Diffusers
18+
trainable: true
19+
enable_gradient_checkpointing_type: full
20+
lora:
21+
enable: true
22+
# rank/alpha and the attention-projection target set follow the
23+
# Wan/Hunyuan LoRA recipes; targets match by substring against
24+
# module names (LTX-2 blocks name theirs to_q/to_k/to_v/to_out).
25+
rank: 16
26+
alpha: 32
27+
target_modules:
28+
- to_q
29+
- to_k
30+
- to_v
31+
- to_out
32+
33+
method:
34+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
35+
36+
training:
37+
distributed:
38+
# The 13B video branch needs sharding even with frozen base weights;
39+
# 4-way HSDP shard matches the validated LTX-2 runs.
40+
num_gpus: 4
41+
sp_size: 1
42+
tp_size: 1
43+
hsdp_replicate_dim: 1
44+
hsdp_shard_dim: 4
45+
46+
data:
47+
# REQUIRED: path to your preprocessed dataset.
48+
data_path: data/your_dataset_preprocessed
49+
dataloader_num_workers: 4
50+
train_batch_size: 1
51+
# LTX2Model requires 0.0: CFG dropout would zero post-connector
52+
# embeddings, which is not the model's unconditional input.
53+
training_cfg_rate: 0.0
54+
seed: 42
55+
# Must match your preprocessed resolution/length.
56+
# num_latent_t = (num_frames - 1) / 8 + 1.
57+
num_latent_t: 11
58+
num_height: 480
59+
num_width: 832
60+
num_frames: 81
61+
62+
optimizer:
63+
# LoRA-only training usually tolerates a higher LR than full
64+
# fine-tuning (the Wan/Hunyuan LoRA recipes use 1.0e-4); tune for
65+
# your dataset.
66+
learning_rate: 1.0e-4
67+
betas: [0.9, 0.999]
68+
weight_decay: 0.0
69+
lr_scheduler: constant
70+
lr_warmup_steps: 0
71+
72+
loop:
73+
# Workload-dependent: set from your dataset size and target epochs.
74+
max_train_steps: 2000
75+
gradient_accumulation_steps: 1
76+
77+
checkpoint:
78+
output_dir: outputs/ltx2_t2v_lora_finetune
79+
training_state_checkpointing_steps: 500
80+
checkpoints_total_limit: 2
81+
resume_from_checkpoint: latest
82+
83+
tracker:
84+
trackers: [wandb]
85+
project_name: fastvideo_ltx2
86+
run_name: ltx2_t2v_lora_finetune
87+
88+
model:
89+
# LTX2Model.predict_noise converts the DiT's x0 output to velocity,
90+
# so the default noise-minus-clean target reproduces the official
91+
# unweighted masked-MSE (mask is all-ones for plain T2V).
92+
precondition_outputs: false
93+
enable_gradient_checkpointing_type: full
94+
95+
callbacks:
96+
grad_clip:
97+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
98+
max_grad_norm: 1.0
99+
validation:
100+
_target_: fastvideo.train.callbacks.validation.ValidationCallback
101+
pipeline_target: fastvideo.pipelines.basic.ltx2.ltx2_pipeline.LTX2Pipeline
102+
# REQUIRED: prompts to sample during training.
103+
dataset_file: data/your_dataset_preprocessed/validation_prompts.json
104+
every_steps: 250
105+
# 8-step single-pass sampling matches the distilled checkpoint
106+
# (validated in the LTX-2 overfit runs).
107+
sampling_steps: [8]
108+
guidance_scale: 1.0
109+
num_frames: 81
110+
111+
# Required so the LTX2T2VConfig pipeline config is resolved from
112+
# init_from (without a `pipeline:` key the loader falls back to a
113+
# generic PipelineConfig and the LTX-2 DiT cannot be constructed).
114+
pipeline: {}

0 commit comments

Comments
 (0)