FastVideo/examples/train/configs/genrl_wan2.1_t2v_1.3B_longcat.yaml at bb5f60e0a1a75cd65cbf82f61dae21749af6544c · Abecid/FastVideo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
# GenRL / Video GRPO: Wan 2.1 T2V 1.3B with LongCat reweighting.
#
# Ported from GenRL/config/longcat.yaml.
#
# - Student: trainable full-parameter model by default.
# - LoRA is still available via models.student.use_lora=true.
# - Full fine-tuning with beta > 0 requires models.reference and much
#   more memory; keep beta at 0.0 for the 4xH100 probe run.
#
# Usage:
#   torchrun --nnodes=1 --nproc_per_node=4 \
#     -m fastvideo.train.entrypoint.train \
#     --config examples/train/configs/genrl_wan2.1_t2v_1.3B_longcat.yaml

models:
  student:
    _target_: fastvideo.train.models.wan.wan_genrl.GenRLWanModel
    init_from: Wan-AI/Wan2.1-T2V-1.3B-Diffusers
    trainable: true
    # Set true for LoRA. LoRA can use method.beta with disable_adapter().
    use_lora: false
    lora_r: 128
    lora_alpha: 64
    lora_init_weights: gaussian
    lora_path: null
    lora_target_modules:
      - to_k
      - to_out
      - to_q
      - to_v
      - ffn.fc_in
      - ffn.fc_out
    enable_gradient_checkpointing_type: full

method:
  _target_: fastvideo.train.methods.rl.genrl.GenRLMethod

  # ---- Reward functions ----
  reward_fn:
    hpsv3_general: 1.0
    hpsv3_percentile: 1.0
    videoalign_mq: 1.0
    videoalign_ta: 1.0
  reward_module: null
  reward_on_gpu: true

  # ---- Data ----
  prompt_dataset_path: GenRL/datasets/filtered_prompts
  prompt_fn: filtered_prompts

  # ---- Sampling ----
  sample_batch_size: 4
  eval_batch_size: 2
  # Sample multiple rollout microbatches, average their PPO losses, then
  # apply one optimizer update. This reduces reward/advantage variance.
  num_batches_per_epoch: 4
  accumulate_ppo_microbatches: true
  eval_every_steps: 20
  eval_num_batches: 1
  eval_num_steps: 16
  eval_guidance_scale: 4.5
  num_inference_steps: 16
  guidance_scale: 4.5
  num_video_per_prompt: 4
  noise_level: 1.0
  sde_type: flow_sde
  sde_window_size: 1
  sde_window_range: [0, 6]
  diffusion_clip: true
  diffusion_clip_value: 0.45
  kl_reward: 0
  same_latent: true

  # ---- Video dimensions ----
  height: 480
  width: 832
  num_frames: 81

  # ---- PPO training ----
  train_batch_size: 4
  num_inner_epochs: 1
  clip_range: 1.0e-4
  adv_clip_max: 5.0
  # Official LoRA LongCat uses beta: 3.0e-4 with disable_adapter().
  # For full fine-tuning on 4 H100s, avoid a second frozen Wan copy.
  beta: 0.0
  use_cfg: true
  # Flash-GRPO-style temporal gradient rectification: avoid the large
  # LongCat sigma/dt multiplier while debugging full-FT stability.
  loss_reweighting: flash_tgr
  loss_reweighting_clip: null
  weight_advantages: true
  # Match official GenRL PPO cadence. With sde_window_size: 1 this is
  # equivalent to one optimizer step per sampled trajectory timestep.
  optimizer_step_per_timestep: true
  log_post_update_kl: true
  max_grad_norm: 1.0
  seed: 42

  # ---- Advantage computation ----
  per_prompt_stat_tracking: true
  global_std: false
  max_group_std: true

training:
  distributed:

    num_gpus: 4
    sp_size: 1
    tp_size: 1
    # Full fine-tuning needs FSDP/HSDP sharding across all 4 GPUs.
    hsdp_replicate_dim: 1
    hsdp_shard_dim: 4

  data:
    # Not used by GenRL (prompt dataloaders are in method config)
    # but required by the config parser.
    data_path: ""
    train_batch_size: 1
    seed: 42
    num_height: 480
    num_width: 832
    num_frames: 81

  optimizer:
    # Full-parameter visual GRPO is much more sensitive than LoRA.
    # DanceGRPO reports 5e-6 to 2e-5 as the practical range.
    learning_rate: 1.0e-5
    betas: [0.9, 0.999]
    weight_decay: 1.0e-4
    lr_scheduler: constant
    lr_warmup_steps: 0

  loop:
    # max_train_steps == num_epochs in GenRL terms.
    max_train_steps: 100000
    gradient_accumulation_steps: 1

  checkpoint:
    output_dir: outputs/genrl_longcat
    training_state_checkpointing_steps: 100
    checkpoints_total_limit: 3

  tracker:
    project_name: VideoRL
    # Leave blank so W&B auto-generates a unique display name per run.
    run_name: ""

  model:
    enable_gradient_checkpointing_type: full

callbacks:
  # Gradnorm call back Disabled; GenRLMethod clips internally.
  ema:
    decay: 0.9
    start_iter: 0
    update_interval: 8
  log_rl_samples:
    every_steps: 1
    max_videos: 4
    fps: 16

pipeline:
  flow_shift: 3.0