Skip to content

Commit e63f63a

Browse files
committed
feat(h3-rest): add one-H100 optimizer smoke config
1 parent 6ed71c9 commit e63f63a

1 file changed

Lines changed: 74 additions & 0 deletions

File tree

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# One-H100 cache-only optimizer correctness gate. Requires the compact cache
2+
# from h3_rest_cache_4gpu_compact.yaml. This is not a quality experiment.
3+
models:
4+
student:
5+
_target_: fastvideo.train.models.minimax_h3.MiniMaxH3RESTModel
6+
init_from: artifacts/rvm_h3/models/fasth3
7+
trainable: true
8+
enable_gradient_checkpointing_type: full
9+
attention_backend: VIDEO_SPARSE_ATTN_H3
10+
student_timesteps: [1000, 750, 500, 250, 0]
11+
verify_cache_hashes: true
12+
lora:
13+
enable: true
14+
rank: 1
15+
alpha: 1
16+
target_modules: [to_q, to_k, to_v, to_out]
17+
18+
method:
19+
_target_: fastvideo.train.methods.knowledge_distillation.H3RESTMethod
20+
attn_kind: vsa
21+
amd_scale: 1.0
22+
amd_bias: 0.5
23+
amd_coefficient_clip: null
24+
audio_loss_weight: 1.0
25+
ema_regularization_weight: 0.2
26+
ema_decay: 0.9
27+
28+
training:
29+
distributed:
30+
num_gpus: 1
31+
sp_size: 1
32+
tp_size: 1
33+
hsdp_replicate_dim: 1
34+
hsdp_shard_dim: 1
35+
data:
36+
data_path: artifacts/rvm_h3/rest_cache/compact
37+
preprocessed_data_type: text_only
38+
dataloader_num_workers: 0
39+
train_batch_size: 1
40+
training_cfg_rate: 0.0
41+
seed: 42
42+
num_latent_t: 12
43+
num_height: 320
44+
num_width: 576
45+
num_frames: 39
46+
optimizer:
47+
learning_rate: 5.0e-6
48+
betas: [0.9, 0.999]
49+
weight_decay: 1.0e-4
50+
lr_scheduler: constant
51+
lr_warmup_steps: 0
52+
loop:
53+
max_train_steps: 1
54+
gradient_accumulation_steps: 1
55+
checkpoint:
56+
output_dir: outputs/h3_rest/1gpu_smoke
57+
training_state_checkpointing_steps: 1
58+
checkpoints_total_limit: 2
59+
tracker:
60+
project_name: fasth3-rest
61+
run_name: h3-rest-1gpu-smoke
62+
model:
63+
precondition_outputs: false
64+
enable_gradient_checkpointing_type: full
65+
vsa_sparsity: 0.9
66+
vsa_cache_tile_buf: false
67+
model_path: artifacts/rvm_h3/models/fasth3
68+
dit_precision: bf16
69+
70+
callbacks:
71+
grad_clip:
72+
_target_: fastvideo.train.callbacks.grad_clip.GradNormClipCallback
73+
max_grad_norm: 1.0
74+
pipeline: {}

0 commit comments

Comments
 (0)