Skip to content

Commit dc8cc41

Browse files
Yanghao-WangYanghao-Wang
committed
[feat] Add MiniMax-H3 Ref2VA and LoRA training support
Add Ref2VA training and enable LoRA training for both T2VA and Ref2VA.
1 parent 7a42851 commit dc8cc41

16 files changed

Lines changed: 1857 additions & 5 deletions

‎examples/train/configs/overfit_minimax_h3_ref2va.yaml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# MiniMax H3 Ref2VA full-tuning overfit experiment on one openhumanvid record.
2+
3+
4+
models:
5+
student:
6+
_target_: fastvideo.train.models.minimax_h3.minimax_h3_ref2va.MiniMaxH3Ref2VAModel
7+
init_from: data/models/MiniMax-H3
8+
trainable: true
9+
enable_gradient_checkpointing_type: full
10+
attention_backend: TORCH_SDPA
11+
12+
method:
13+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
14+
15+
training:
16+
distributed:
17+
num_gpus: 64
18+
sp_size: 8
19+
tp_size: 1
20+
hsdp_replicate_dim: 8
21+
hsdp_shard_dim: 8
22+
pin_cpu_memory: true
23+
24+
data:
25+
data_path:
26+
data/openhumanvid_h3_ref2va_single_sample_preprocessed: 8
27+
preprocessed_data_type: t2va
28+
dataloader_num_workers: 0
29+
train_batch_size: 1
30+
training_cfg_rate: 0.0
31+
seed: 42
32+
num_latent_t: 37
33+
num_height: 768
34+
num_width: 1344
35+
num_frames: 124
36+
37+
optimizer:
38+
learning_rate: 5.0e-5
39+
betas: [0.9, 0.999]
40+
weight_decay: 0.0
41+
lr_scheduler: constant
42+
lr_warmup_steps: 0
43+
44+
loop:
45+
max_train_steps: 400
46+
gradient_accumulation_steps: 1
47+
48+
checkpoint:
49+
output_dir: runs/minimax_h3_ref2va_openhumanvid_single_sample_overfit/checkpoints
50+
training_state_checkpointing_steps: 20
51+
checkpoints_total_limit: 2
52+
resume_from_checkpoint: ""
53+
54+
tracker:
55+
trackers: [wandb]
56+
project_name: fastvideo_minimax_h3
57+
run_name: minimax_h3_ref2va_openhumanvid_single_sample_overfit
58+
59+
model:
60+
precondition_outputs: false
61+
enable_gradient_checkpointing_type: full
62+
63+
dit_precision: bf16
64+
65+
callbacks:
66+
grad_clip:
67+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
68+
max_grad_norm: 1.0
69+
validation:
70+
_target_: fastvideo.train.callbacks.minimax_h3_ref2va_validation.MiniMaxH3Ref2VAValidationCallback
71+
pipeline_target: fastvideo.pipelines.basic.minimax_h3.minimax_h3_pipeline.MiniMaxH3Ref2VAModularPipeline
72+
dataset_file: examples/training/finetune/minimax-h3/openhumanvid/validation.json
73+
every_steps: 20
74+
run_at_start: true
75+
sampling_steps: [50]
76+
guidance_scale: 1.0
77+
num_frames: 124
78+
num_videos_per_prompt: 1
79+
use_validation_media_conditioning: false
80+
offload_training_state: true
81+
text_encoder_cpu_offload: true
82+
vae_cpu_offload: true
83+
84+
pipeline: {}

‎examples/train/configs/overfit_minimax_h3_ref2va_lora.yaml‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# MiniMax H3 Ref2VA full-tuning overfit experiment on one openhumanvid record.
2+
3+
4+
models:
5+
student:
6+
_target_: fastvideo.train.models.minimax_h3.minimax_h3_ref2va.MiniMaxH3Ref2VALoraModel
7+
init_from: data/models/MiniMax-H3
8+
trainable: true
9+
enable_gradient_checkpointing_type: full
10+
attention_backend: TORCH_SDPA
11+
lora:
12+
enable: true
13+
rank: 32
14+
alpha: 32
15+
target_modules:
16+
- attn.to_q
17+
- attn.to_k
18+
- attn.to_v
19+
- attn.to_out
20+
- ff.fc_in
21+
- ff.fc_out
22+
expected_lora_layers: 312
23+
24+
method:
25+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
26+
27+
training:
28+
distributed:
29+
num_gpus: 64
30+
sp_size: 8
31+
tp_size: 1
32+
hsdp_replicate_dim: 8
33+
hsdp_shard_dim: 8
34+
pin_cpu_memory: true
35+
36+
data:
37+
data_path:
38+
data/openhumanvid_h3_ref2va_single_sample_preprocessed: 8
39+
preprocessed_data_type: t2va
40+
dataloader_num_workers: 0
41+
train_batch_size: 1
42+
training_cfg_rate: 0.0
43+
seed: 42
44+
num_latent_t: 37
45+
num_height: 768
46+
num_width: 1344
47+
num_frames: 124
48+
49+
optimizer:
50+
learning_rate: 5.0e-5
51+
betas: [0.9, 0.999]
52+
weight_decay: 0.0
53+
lr_scheduler: constant
54+
lr_warmup_steps: 0
55+
56+
loop:
57+
max_train_steps: 400
58+
gradient_accumulation_steps: 1
59+
60+
checkpoint:
61+
output_dir: runs/minimax_h3_ref2va_openhumanvid_single_sample_overfit_lora_rank32/checkpoints
62+
training_state_checkpointing_steps: 20
63+
checkpoints_total_limit: 2
64+
resume_from_checkpoint: ""
65+
66+
tracker:
67+
trackers: [wandb]
68+
project_name: fastvideo_minimax_h3
69+
run_name: minimax_h3_ref2va_openhumanvid_single_sample_overfit_lora_rank32
70+
71+
model:
72+
precondition_outputs: false
73+
enable_gradient_checkpointing_type: full
74+
75+
dit_precision: bf16
76+
77+
callbacks:
78+
grad_clip:
79+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
80+
max_grad_norm: 1.0
81+
validation:
82+
_target_: fastvideo.train.callbacks.minimax_h3_ref2va_validation.MiniMaxH3Ref2VAValidationCallback
83+
pipeline_target: fastvideo.pipelines.basic.minimax_h3.minimax_h3_pipeline.MiniMaxH3Ref2VAModularPipeline
84+
dataset_file: examples/training/finetune/minimax-h3/openhumanvid/validation.json
85+
every_steps: 20
86+
run_at_start: true
87+
sampling_steps: [50]
88+
guidance_scale: 1.0
89+
num_frames: 124
90+
num_videos_per_prompt: 1
91+
use_validation_media_conditioning: false
92+
offload_training_state: true
93+
text_encoder_cpu_offload: true
94+
vae_cpu_offload: true
95+
96+
pipeline: {}

‎examples/train/configs/overfit_minimax_h3_t2va_lora.yaml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# MiniMax H3 T2VA 400-step overfit experiment on one Crush-Smol record.
2+
#
3+
# Submit eight H200 nodes through the fixed launch script:
4+
# bash examples/train/launch_minimax_h3_t2va_crush_smol_validation.sh
5+
6+
models:
7+
student:
8+
_target_: fastvideo.train.models.minimax_h3.minimax_h3.MiniMaxH3LoraModel
9+
init_from: data/models/MiniMax-H3
10+
trainable: true
11+
enable_gradient_checkpointing_type: full
12+
attention_backend: TORCH_SDPA
13+
lora:
14+
enable: true
15+
rank: 32
16+
alpha: 32
17+
target_modules:
18+
- attn.to_q
19+
- attn.to_k
20+
- attn.to_v
21+
- attn.to_out
22+
- ff.fc_in
23+
- ff.fc_out
24+
expected_lora_layers: 312
25+
26+
method:
27+
_target_: fastvideo.train.methods.fine_tuning.finetune.FineTuneMethod
28+
29+
training:
30+
distributed:
31+
num_gpus: 64
32+
sp_size: 8
33+
tp_size: 1
34+
hsdp_replicate_dim: 8
35+
hsdp_shard_dim: 8
36+
pin_cpu_memory: true
37+
38+
data:
39+
data_path:
40+
data/crush-smol_h3_t2va_single_sample_preprocessed: 8
41+
preprocessed_data_type: t2va
42+
dataloader_num_workers: 0
43+
train_batch_size: 1
44+
training_cfg_rate: 0.0
45+
seed: 42
46+
num_latent_t: 37
47+
num_height: 768
48+
num_width: 1344
49+
num_frames: 124
50+
51+
optimizer:
52+
learning_rate: 5.0e-5
53+
betas: [0.9, 0.999]
54+
weight_decay: 0.0
55+
lr_scheduler: constant
56+
lr_warmup_steps: 0
57+
58+
loop:
59+
max_train_steps: 400
60+
gradient_accumulation_steps: 1
61+
62+
checkpoint:
63+
output_dir: runs/minimax_h3_t2va_crush_smol_single_sample_overfit_lora_rank32/checkpoints
64+
training_state_checkpointing_steps: 20
65+
checkpoints_total_limit: 2
66+
resume_from_checkpoint: ""
67+
68+
tracker:
69+
trackers: [wandb]
70+
project_name: fastvideo_minimax_h3
71+
run_name: minimax_h3_t2va_crush_smol_single_sample_overfit_lora_rank32
72+
73+
model:
74+
precondition_outputs: false
75+
enable_gradient_checkpointing_type: full
76+
77+
dit_precision: bf16
78+
79+
callbacks:
80+
grad_clip:
81+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
82+
max_grad_norm: 1.0
83+
validation:
84+
_target_: fastvideo.train.callbacks.validation.ValidationCallback
85+
pipeline_target: fastvideo.pipelines.basic.minimax_h3.minimax_h3_pipeline.MiniMaxH3Pipeline
86+
dataset_file: examples/training/finetune/Wan2.1-Fun-1.3B-InP/crush_smol/validation.json
87+
every_steps: 20
88+
run_at_start: true
89+
sampling_steps: [50]
90+
guidance_scale: 1.0
91+
num_frames: 124
92+
num_videos_per_prompt: 1
93+
use_validation_media_conditioning: false
94+
offload_training_state: true
95+
text_encoder_cpu_offload: true
96+
vae_cpu_offload: true
97+
98+
pipeline: {}

‎examples/train/setup_minimax_h3_ref2va_openhumanvid_single_sample.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
# Encode every OpenHumanVid record into MiniMax H3 Ref2VA training data.
3+
4+
set -euo pipefail
5+
set +x
6+
7+
WORKTREE_ROOT=/mnt/weka/shrd/wm/junda/fv-hub/fastvideo-add-h3-sft
8+
ENV_FILE=/mnt/weka/shrd/wm/junda/fv-hub/.env
9+
VENV_BIN=/mnt/weka/shrd/wm/junda/fv-hub/.venv/bin
10+
MANIFEST="${WORKTREE_ROOT}/examples/training/finetune/minimax-h3/openhumanvid/train.jsonl"
11+
MODEL_DIR="${WORKTREE_ROOT}/data/models/MiniMax-H3"
12+
MODEL_INDEX="${MODEL_DIR}/model_index.json"
13+
TRANSFORMER_REF_DIR="${MODEL_DIR}/transformer_ref"
14+
OUTPUT_DIR="${WORKTREE_ROOT}/data/openhumanvid_h3_ref2va_single_sample_preprocessed"
15+
16+
[[ -f "${ENV_FILE}" ]] || { echo "Missing environment file: ${ENV_FILE}" >&2; exit 1; }
17+
[[ -f "${MANIFEST}" ]] || { echo "Missing OpenHumanVid training manifest: ${MANIFEST}" >&2; exit 1; }
18+
[[ -f "${MODEL_INDEX}" ]] || { echo "Missing MiniMax H3 checkpoint: ${MODEL_INDEX}" >&2; exit 1; }
19+
[[ -d "${TRANSFORMER_REF_DIR}" ]] || { echo "Missing MiniMax H3 transformer_ref: ${TRANSFORMER_REF_DIR}" >&2; exit 1; }
20+
[[ -x "${VENV_BIN}/python" ]] || { echo "Missing preprocessing environment: ${VENV_BIN}" >&2; exit 1; }
21+
[[ -x "${VENV_BIN}/torchrun" ]] || { echo "Missing preprocessing environment: ${VENV_BIN}" >&2; exit 1; }
22+
23+
set -a
24+
source "${ENV_FILE}"
25+
set +a
26+
27+
cd "${WORKTREE_ROOT}"
28+
"${VENV_BIN}/python" \
29+
-m fastvideo.pipelines.preprocess.preprocess_minimax_h3_ref2va \
30+
--manifest "${MANIFEST}" \
31+
--validate-manifest-only
32+
33+
CUDA_VISIBLE_DEVICES=0 "${VENV_BIN}/torchrun" \
34+
--standalone \
35+
--nnodes=1 \
36+
--nproc-per-node=1 \
37+
-m fastvideo.pipelines.preprocess.preprocess_minimax_h3_ref2va \
38+
--manifest "${MANIFEST}" \
39+
--model-path "${MODEL_DIR}" \
40+
--output-dir "${OUTPUT_DIR}"
41+
42+
"${VENV_BIN}/python" \
43+
-m fastvideo.pipelines.preprocess.preprocess_minimax_h3_ref2va \
44+
--manifest "${MANIFEST}" \
45+
--output-dir "${OUTPUT_DIR}" \
46+
--validate-only
47+
48+
echo "Prepared OpenHumanVid MiniMax H3 Ref2VA training data in ${OUTPUT_DIR}"

‎examples/training/finetune/minimax-h3/openhumanvid/refs/audios/train_000001.wav‎

260 KB
Binary file not shown.

‎examples/training/finetune/minimax-h3/openhumanvid/refs/images/train_000001.png‎

715 KB
Loading

‎examples/training/finetune/minimax-h3/openhumanvid/targets/train_000001.mp4‎

1.41 MB
Binary file not shown.

‎examples/training/finetune/minimax-h3/openhumanvid/train.jsonl‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"schema_version":"minimax_h3_ref2va_raw_v1",
3+
"id":"train_000001",
4+
"target":{"video":"targets/train_000001.mp4"},
5+
"caption":"The video captures a young adult, likely in their twenties, seated and animatedly gesturing with their hands as they converse with others at a festive indoor gathering, surrounded by a brick wall adorned with Christmas decorations. The individual, wearing a brown turtleneck sweater and dark hair, is flanked by others, including a person in a red coat and a black jacket, in a warm and cozy atmosphere.",
6+
"references":[
7+
{"type":"image","image":"refs/images/train_000001.png"},
8+
{"type":"audio","audio":"refs/audios/train_000001.wav"}
9+
]
10+
}

‎examples/training/finetune/minimax-h3/openhumanvid/validation.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"data": [
3+
{
4+
"schema_version": "minimax_h3_ref2va_raw_v1",
5+
"id": "val_000001",
6+
"target": {"video": "targets/train_000001.mp4"},
7+
"caption": "The video captures a young adult, likely in their twenties, seated and animatedly gesturing with their hands as they converse with others at a festive indoor gathering, surrounded by a brick wall adorned with Christmas decorations. The individual, wearing a brown turtleneck sweater and dark hair, is flanked by others, including a person in a red coat and a black jacket, in a warm and cozy atmosphere.",
8+
"references": [
9+
{
10+
"type": "image",
11+
"image": "refs/images/train_000001.png"
12+
},
13+
{
14+
"type": "audio",
15+
"audio": "refs/audios/train_000001.wav"
16+
}
17+
]
18+
}
19+
]
20+
}

0 commit comments

Comments
 (0)