Skip to content

Commit 88ec837

Browse files
yaoyu-33yaoyu-33
committed
[build] chore: bump MCore main to 616c2fd
Signed-off-by: Yu Yao <yaoyu.094@gmail.com>
2 parents 56f0f3e + c30562d commit 88ec837

14 files changed

Lines changed: 254 additions & 86 deletions

File tree

‎.main.commit‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7f7c424fcf9a50ddc06832509e1d5be6ebda0868
1+
616c2fd86b0f29fe10bc6cbec3653e57ac07b4d5

‎3rdparty/Megatron-LM‎

Submodule Megatron-LM updated 56 files

‎docker/Dockerfile.ci‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ finally:
4545
PY
4646

4747
ARG INSTALL_DEEPEP=True
48-
ARG DEEPEP_COMMIT=34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72
48+
ARG DEEPEP_COMMIT=10d4dd7377d5bce900fbb4b80cce863764892b95
4949
ARG REINSTALL_NVSHMEM=True
5050

5151
ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \

‎src/megatron/bridge/models/conversion/utils.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,18 @@ def unwrap_model(model, module_instances=None):
4646
if module_instances is None:
4747
from megatron.core.distributed import DistributedDataParallel as DDP
4848
from megatron.core.distributed import TorchFullyShardedDataParallel as torch_FSDP
49+
from megatron.core.distributed.fsdp.mcore_fsdp_adapter import (
50+
FullyShardedDataParallelV1,
51+
FullyShardedDataParallelV2,
52+
)
4953
from megatron.core.distributed.fsdp.src.megatron_fsdp.megatron_fsdp import MegatronFSDP
5054
from megatron.core.transformer.module import Float16Module
5155

52-
from megatron.bridge.training.fsdp_compat import MEGATRON_FSDP_TYPES
53-
5456
module_instances = (
5557
DDP,
5658
torch_FSDP,
57-
*MEGATRON_FSDP_TYPES,
59+
FullyShardedDataParallelV1,
60+
FullyShardedDataParallelV2,
5861
Float16Module,
5962
MegatronFSDP,
6063
)

‎src/megatron/bridge/recipes/qwen_vl/h100/qwen35_vl.py‎

Lines changed: 71 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,33 @@
3939
from megatron.bridge.utils.cuda_graph import clear_cuda_graph_modules, set_cuda_graph_modules
4040

4141

42+
def _enable_gdn_conv_fusion(model) -> None:
43+
"""Enable the GatedDeltaNet pre-gated-delta-rule fusion on a Qwen3.5-VL model.
44+
45+
Fuses the pre-gated-delta-rule path (causal conv1d + QK L2-norm) into Triton
46+
kernels. Qwen3.5-VL runs 3 of every 4 layers as GDN -- 30 of 40 at 35B-A3B,
47+
45 of 60 at 397B-A17B -- so the fused path covers most of the step. Measured
48+
+5.8% on 8x GB300 with real DataComp Energon (BF16, MBS4/GBS512, EP8, steps
49+
101-150, n=50): 20332.0 -> 19214.8 ms/step, 297.0 -> 314.2 TFLOP/s/GPU.
50+
51+
Requires a Megatron-Core carrying ``gdn_pre_gated_delta_rule_fusion``.
52+
Assigning an unknown field on the model config does NOT raise -- it would
53+
silently create an unused attribute, leaving the recipe looking enabled
54+
while running unfused -- so the guard is deliberate.
55+
"""
56+
if hasattr(type(model), "gdn_pre_gated_delta_rule_fusion") or hasattr(model, "gdn_pre_gated_delta_rule_fusion"):
57+
model.gdn_pre_gated_delta_rule_fusion = True
58+
59+
60+
def _qwen35_vl_provider(hf_path: str):
61+
"""Build the Qwen3.5/Qwen3.6-VL Megatron provider.
62+
63+
Construction only -- shared so the AutoBridge boilerplate is written once.
64+
Performance defaults are applied by the recipes, not here.
65+
"""
66+
return AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
67+
68+
4269
def _apply_qwen35_vl_35b_a3b_16gpu_h100_execution_config(cfg: ConfigContainer) -> None:
4370
"""Apply the measured 16-H100 execution policy for Qwen3.5/Qwen3.6-VL 35B-A3B."""
4471
cfg.model.tensor_model_parallel_size = 1
@@ -114,7 +141,8 @@ def qwen35_vl_9b_pretrain_4gpu_h100_bf16_mock_config() -> ConfigContainer:
114141
cfg = _pretrain_common()
115142

116143
hf_path = "Qwen/Qwen3.5-9B"
117-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
144+
cfg.model = _qwen35_vl_provider(hf_path)
145+
_enable_gdn_conv_fusion(cfg.model)
118146
cfg.model.tensor_model_parallel_size = 4
119147
cfg.model.pipeline_model_parallel_size = 1
120148
cfg.model.pipeline_dtype = None
@@ -162,7 +190,8 @@ def qwen35_vl_27b_pretrain_16gpu_h100_bf16_mock_config() -> ConfigContainer:
162190
cfg = _pretrain_common()
163191

164192
hf_path = "Qwen/Qwen3.5-27B"
165-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
193+
cfg.model = _qwen35_vl_provider(hf_path)
194+
_enable_gdn_conv_fusion(cfg.model)
166195
cfg.model.tensor_model_parallel_size = 4
167196
cfg.model.pipeline_model_parallel_size = 4
168197
cfg.model.pipeline_dtype = torch.bfloat16
@@ -216,7 +245,8 @@ def qwen35_vl_35b_a3b_pretrain_8gpu_h100_bf16_mock_config() -> ConfigContainer:
216245
cfg = _pretrain_common()
217246

218247
hf_path = "Qwen/Qwen3.5-35B-A3B"
219-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
248+
cfg.model = _qwen35_vl_provider(hf_path)
249+
_enable_gdn_conv_fusion(cfg.model)
220250
cfg.model.tensor_model_parallel_size = 4
221251
cfg.model.pipeline_model_parallel_size = 2
222252
cfg.model.pipeline_dtype = torch.bfloat16
@@ -334,7 +364,8 @@ def qwen35_vl_122b_a10b_pretrain_128gpu_h100_bf16_mock_config() -> ConfigContain
334364
cfg = _pretrain_common()
335365

336366
hf_path = "Qwen/Qwen3.5-122B-A10B"
337-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
367+
cfg.model = _qwen35_vl_provider(hf_path)
368+
_enable_gdn_conv_fusion(cfg.model)
338369
cfg.model.tensor_model_parallel_size = 4
339370
cfg.model.pipeline_model_parallel_size = 8
340371
cfg.model.pipeline_dtype = torch.bfloat16
@@ -385,7 +416,8 @@ def qwen35_vl_397b_a17b_pretrain_512gpu_h100_bf16_mock_config() -> ConfigContain
385416
cfg = _pretrain_common()
386417

387418
hf_path = "Qwen/Qwen3.5-397B-A17B"
388-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
419+
cfg.model = _qwen35_vl_provider(hf_path)
420+
_enable_gdn_conv_fusion(cfg.model)
389421
cfg.model.tensor_model_parallel_size = 4
390422
cfg.model.pipeline_model_parallel_size = 16
391423
cfg.model.pipeline_dtype = torch.bfloat16
@@ -450,7 +482,8 @@ def qwen35_vl_800m_sft_1gpu_h100_bf16_config() -> ConfigContainer:
450482

451483
# Model config
452484
hf_path = "Qwen/Qwen3.5-0.8B"
453-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
485+
cfg.model = _qwen35_vl_provider(hf_path)
486+
_enable_gdn_conv_fusion(cfg.model)
454487
cfg.model.seq_length = 4096
455488

456489
# Parallelism settings
@@ -550,7 +583,8 @@ def qwen35_vl_2b_sft_1gpu_h100_bf16_config() -> ConfigContainer:
550583

551584
# Model config
552585
hf_path = "Qwen/Qwen3.5-2B"
553-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
586+
cfg.model = _qwen35_vl_provider(hf_path)
587+
_enable_gdn_conv_fusion(cfg.model)
554588
cfg.model.seq_length = 4096
555589

556590
# Parallelism settings
@@ -650,7 +684,8 @@ def qwen35_vl_4b_sft_2gpu_h100_bf16_config() -> ConfigContainer:
650684

651685
# Model config
652686
hf_path = "Qwen/Qwen3.5-4B"
653-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
687+
cfg.model = _qwen35_vl_provider(hf_path)
688+
_enable_gdn_conv_fusion(cfg.model)
654689
cfg.model.seq_length = 4096
655690

656691
# Parallelism settings
@@ -750,7 +785,8 @@ def qwen35_vl_9b_sft_4gpu_h100_bf16_config() -> ConfigContainer:
750785

751786
# Model config
752787
hf_path = "Qwen/Qwen3.5-9B"
753-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
788+
cfg.model = _qwen35_vl_provider(hf_path)
789+
_enable_gdn_conv_fusion(cfg.model)
754790
cfg.model.seq_length = 4096
755791

756792
# Parallelism settings
@@ -848,7 +884,8 @@ def qwen35_vl_27b_sft_16gpu_h100_bf16_config() -> ConfigContainer:
848884

849885
# Model config
850886
hf_path = "Qwen/Qwen3.5-27B"
851-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
887+
cfg.model = _qwen35_vl_provider(hf_path)
888+
_enable_gdn_conv_fusion(cfg.model)
852889
cfg.model.seq_length = 4096
853890

854891
# Parallelism settings
@@ -952,7 +989,8 @@ def _qwen35_vl_35b_a3b_sft_base_config() -> ConfigContainer:
952989

953990
# Model config
954991
hf_path = "Qwen/Qwen3.5-35B-A3B"
955-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
992+
cfg.model = _qwen35_vl_provider(hf_path)
993+
_enable_gdn_conv_fusion(cfg.model)
956994
cfg.model.seq_length = 4096
957995

958996
# Parallelism settings
@@ -1161,7 +1199,8 @@ def qwen35_vl_35b_a3b_sft_2gpu_h100_bf16_fsdp_config() -> ConfigContainer:
11611199

11621200
# Model config
11631201
hf_path = "Qwen/Qwen3.5-35B-A3B"
1164-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1202+
cfg.model = _qwen35_vl_provider(hf_path)
1203+
_enable_gdn_conv_fusion(cfg.model)
11651204
cfg.model.seq_length = 4096
11661205

11671206
# Parallelism settings
@@ -1283,7 +1322,8 @@ def qwen35_vl_122b_a10b_sft_48gpu_h100_bf16_config() -> ConfigContainer:
12831322

12841323
# Model config
12851324
hf_path = "Qwen/Qwen3.5-122B-A10B"
1286-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1325+
cfg.model = _qwen35_vl_provider(hf_path)
1326+
_enable_gdn_conv_fusion(cfg.model)
12871327
cfg.model.seq_length = 4096
12881328

12891329
# Parallelism settings
@@ -1395,7 +1435,8 @@ def qwen35_vl_397b_a17b_sft_128gpu_h100_bf16_config() -> ConfigContainer:
13951435

13961436
# Model config
13971437
hf_path = "Qwen/Qwen3.5-397B-A17B"
1398-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1438+
cfg.model = _qwen35_vl_provider(hf_path)
1439+
_enable_gdn_conv_fusion(cfg.model)
13991440
cfg.model.seq_length = 4096
14001441

14011442
# Parallelism settings
@@ -1515,7 +1556,8 @@ def qwen35_vl_800m_peft_1gpu_h100_bf16_config(
15151556

15161557
# Model config
15171558
hf_path = "Qwen/Qwen3.5-0.8B"
1518-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1559+
cfg.model = _qwen35_vl_provider(hf_path)
1560+
_enable_gdn_conv_fusion(cfg.model)
15191561
cfg.model.seq_length = 4096
15201562

15211563
# Parallelism settings
@@ -1616,7 +1658,8 @@ def qwen35_vl_2b_peft_1gpu_h100_bf16_config(
16161658

16171659
# Model config
16181660
hf_path = "Qwen/Qwen3.5-2B"
1619-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1661+
cfg.model = _qwen35_vl_provider(hf_path)
1662+
_enable_gdn_conv_fusion(cfg.model)
16201663
cfg.model.seq_length = 4096
16211664

16221665
# Parallelism settings
@@ -1717,7 +1760,8 @@ def qwen35_vl_4b_peft_1gpu_h100_bf16_config(
17171760

17181761
# Model config
17191762
hf_path = "Qwen/Qwen3.5-4B"
1720-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1763+
cfg.model = _qwen35_vl_provider(hf_path)
1764+
_enable_gdn_conv_fusion(cfg.model)
17211765
cfg.model.seq_length = 4096
17221766

17231767
# Parallelism settings
@@ -1818,7 +1862,8 @@ def qwen35_vl_9b_peft_1gpu_h100_bf16_config(
18181862

18191863
# Model config
18201864
hf_path = "Qwen/Qwen3.5-9B"
1821-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1865+
cfg.model = _qwen35_vl_provider(hf_path)
1866+
_enable_gdn_conv_fusion(cfg.model)
18221867
cfg.model.seq_length = 4096
18231868

18241869
# Parallelism settings
@@ -1919,7 +1964,8 @@ def qwen35_vl_27b_peft_2gpu_h100_bf16_config(
19191964

19201965
# Model config
19211966
hf_path = "Qwen/Qwen3.5-27B"
1922-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
1967+
cfg.model = _qwen35_vl_provider(hf_path)
1968+
_enable_gdn_conv_fusion(cfg.model)
19231969
cfg.model.seq_length = 4096
19241970

19251971
# Parallelism settings
@@ -2025,7 +2071,8 @@ def qwen35_vl_35b_a3b_peft_4gpu_h100_bf16_config(
20252071

20262072
# Model config
20272073
hf_path = "Qwen/Qwen3.5-35B-A3B"
2028-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
2074+
cfg.model = _qwen35_vl_provider(hf_path)
2075+
_enable_gdn_conv_fusion(cfg.model)
20292076
cfg.model.seq_length = 4096
20302077

20312078
# Parallelism settings
@@ -2167,7 +2214,8 @@ def qwen35_vl_122b_a10b_peft_8gpu_h100_bf16_config(
21672214

21682215
# Model config
21692216
hf_path = "Qwen/Qwen3.5-122B-A10B"
2170-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
2217+
cfg.model = _qwen35_vl_provider(hf_path)
2218+
_enable_gdn_conv_fusion(cfg.model)
21712219
cfg.model.seq_length = 4096
21722220

21732221
# Parallelism settings
@@ -2282,7 +2330,8 @@ def qwen35_vl_397b_a17b_peft_32gpu_h100_bf16_config(
22822330

22832331
# Model config
22842332
hf_path = "Qwen/Qwen3.5-397B-A17B"
2285-
cfg.model = AutoBridge.from_hf_pretrained(hf_path).to_megatron_provider(load_weights=False)
2333+
cfg.model = _qwen35_vl_provider(hf_path)
2334+
_enable_gdn_conv_fusion(cfg.model)
22862335
cfg.model.seq_length = 4096
22872336

22882337
# Parallelism settings

‎src/megatron/bridge/training/config.py‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@
7171
from megatron.bridge.peft.base import PEFT
7272
from megatron.bridge.training.comm_overlap import CommOverlapConfig
7373
from megatron.bridge.training.flex_dispatcher_backend import validate_flex_dispatcher_backend
74-
from megatron.bridge.training.fsdp_compat import MCORE_HAS_MEGATRON_FSDP_V2
7574
from megatron.bridge.training.mixed_precision import MixedPrecisionConfig, get_mixed_precision_config
7675
from megatron.bridge.training.tokenizers.config import TokenizerConfig
7776
from megatron.bridge.training.utils.config_utils import _ConfigContainerBase as Container
@@ -1082,8 +1081,8 @@ def _validate_and_apply_megatron_fsdp_configs(self) -> None:
10821081
self.dist.use_megatron_fsdp = True
10831082
self.ddp.use_megatron_fsdp = True
10841083

1085-
megatron_fsdp_version = getattr(self.ddp, "megatron_fsdp_version", 1)
1086-
if not MCORE_HAS_MEGATRON_FSDP_V2 or megatron_fsdp_version == 1:
1084+
megatron_fsdp_version = self.ddp.megatron_fsdp_version
1085+
if megatron_fsdp_version == 1:
10871086
self._validate_and_apply_megatron_fsdp_v1_configs()
10881087
elif megatron_fsdp_version == 2:
10891088
self._validate_and_apply_megatron_fsdp_v2_configs()
@@ -1173,32 +1172,20 @@ def _validate_and_apply_megatron_fsdp_v2_configs(self) -> None:
11731172
raise ValueError("MFSDP V2 does not support use_tp_pp_dp_mapping.")
11741173
if self.rng.data_parallel_random_init:
11751174
raise ValueError("MFSDP V2 does not support data_parallel_random_init.")
1176-
if self.ddp.num_distributed_optimizer_instances != 1:
1177-
raise ValueError("MFSDP V2 does not currently support HSDP.")
1178-
if self.ddp.outer_dp_sharding_strategy != "no_shard":
1179-
raise ValueError("MFSDP V2 does not currently support outer DP sharding.")
11801175
if self.checkpoint.save is not None or self.checkpoint.load is not None:
11811176
raise ValueError("MFSDP V2 checkpoint save and load are not yet supported.")
11821177
if self.checkpoint.pretrained_checkpoint is not None:
11831178
raise ValueError("MFSDP V2 checkpoint loading is not yet supported.")
11841179
if self.optimizer.loss_scale is not None:
11851180
raise ValueError("MFSDP V2 does not support loss scaling.")
1186-
if self.optimizer.clip_grad > 0.0:
1187-
raise ValueError("MFSDP V2 does not currently support gradient clipping.")
1188-
if self.optimizer.use_precision_aware_optimizer:
1189-
raise ValueError("MFSDP V2 does not support precision-aware optimizer.")
11901181
if self.optimizer.optimizer_cpu_offload:
11911182
raise ValueError("MFSDP V2 does not support optimizer CPU offload.")
11921183
if self.optimizer.use_layer_wise_distributed_optimizer:
11931184
raise ValueError("MFSDP V2 does not support layer-wise distributed optimizer.")
1194-
if self.optimizer.optimizer_cuda_graph:
1195-
raise ValueError("MFSDP V2 does not support optimizer CUDA graphs.")
11961185
if self.model.calculate_per_token_loss:
11971186
raise ValueError("MFSDP V2 does not support per-token loss normalization.")
11981187
if self.model.fp8 or self.model.fp4 or self.ddp.fp8_param_gather or self.ddp.fp4_param_gather:
11991188
raise ValueError("MFSDP V2 does not support FP8 or FP4.")
1200-
if self.model.cuda_graph_impl != "none" or self.ddp.megatron_fsdp_cuda_graph_mode:
1201-
raise ValueError("MFSDP V2 does not support CUDA graphs.")
12021189

12031190
self.ddp.data_parallel_sharding_strategy = "optim_grads_params"
12041191
self.ddp.use_distributed_optimizer = False

‎src/megatron/bridge/training/fsdp_compat.py‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

0 commit comments

Comments
 (0)