File tree Expand file tree Collapse file tree
src/megatron/bridge/recipes/qwen Expand file tree Collapse file tree Original file line number Diff line number Diff line change 6565 qwen3_32b_sft_config ,
6666 qwen3_600m_peft_config ,
6767 qwen3_600m_pretrain_config ,
68+ qwen3_600m_sft_128k_config ,
6869 qwen3_600m_sft_config ,
69- qwen3_600m_sft_yarn_128k_config ,
7070)
7171
7272# Qwen3 MoE models
128128 "qwen3_14b_pretrain_config" ,
129129 "qwen3_32b_pretrain_config" ,
130130 "qwen3_600m_sft_config" ,
131- "qwen3_600m_sft_yarn_128k_config " ,
131+ "qwen3_600m_sft_128k_config " ,
132132 "qwen3_1p7b_sft_config" ,
133133 "qwen3_4b_sft_config" ,
134134 "qwen3_8b_sft_config" ,
Original file line number Diff line number Diff line change @@ -587,7 +587,7 @@ def qwen3_600m_sft_config() -> ConfigContainer:
587587 return cfg
588588
589589
590- def qwen3_600m_sft_yarn_128k_config () -> ConfigContainer :
590+ def qwen3_600m_sft_128k_config () -> ConfigContainer :
591591 """Return a full SFT config for Qwen3 600M with YaRN 128K context extension.
592592
593593 Extends the base 600M SFT config with YaRN RoPE scaling to support 128K sequence
@@ -611,13 +611,10 @@ def qwen3_600m_sft_yarn_128k_config() -> ConfigContainer:
611611 cfg .model .context_parallel_size = 8
612612 cfg .model .sequence_parallel = False
613613
614- # 128K sequence length (must be set on both model and dataset)
614+ # 128K sequence length
615615 cfg .model .seq_length = 131072
616616 cfg .dataset .seq_length = 131072
617617 cfg .dataset .packed_sequence_specs .packed_sequence_size = 131072
618- # cfg.model.seq_length = 4096
619- # cfg.dataset.seq_length = 4096
620- # cfg.dataset.packed_sequence_specs.packed_sequence_size = 4096
621618 # Set pad_seq_to_mult for context parallelism
622619 cfg .dataset .packed_sequence_specs .pad_seq_to_mult = cfg .model .context_parallel_size * 2
623620
You can’t perform that action at this time.
0 commit comments