2626import torch .nn as nn
2727from megatron .core .models .hybrid .hybrid_layer_allocation import Symbols , parse_hybrid_pattern
2828from megatron .core .num_microbatches_calculator import get_num_microbatches
29+ from megatron .core .models .hybrid .hybrid_layer_allocation import Symbols
2930from megatron .core .tensor_parallel import param_is_not_tensor_parallel_duplicate
3031from megatron .core .transformer .module import MegatronModule
3132from megatron .core .transformer .moe .moe_utils import track_moe_metrics
@@ -1023,6 +1024,17 @@ def training_log(
10231024 if getattr (config .model , "moe_z_loss_coeff" , None ) is not None :
10241025 track_names .append ("z_loss" )
10251026
1027+ mtp_num_layers = getattr (config .model , "mtp_num_layers" , None )
1028+ if getattr (config .model , "is_hybrid_model" , False ):
1029+ # MCore expects main and MTP MoE layer counts separately.
1030+ pattern = getattr (config .model , "hybrid_layer_pattern" , "" )
1031+ main_pattern , mtp_separator , mtp_patterns = pattern .partition (Symbols .MTP_SEPARATOR )
1032+ layers = main_pattern .count (Symbols .MOE )
1033+ if mtp_separator :
1034+ mtp_num_layers = mtp_patterns .count (Symbols .MOE )
1035+ else :
1036+ layers = getattr (config .model , "num_layers" , None )
1037+
10261038 # Wrap the TB writer so MoE/MTP metrics also reach MLFlow / Comet (issue #2989).
10271039 # No-op when neither logger is configured: the original writer is returned as-is.
10281040 moe_metric_writer = _build_moe_metric_writer (writer , comet_logger , mlflow_logger )
@@ -1039,7 +1051,7 @@ def training_log(
10391051 num_layers = config .model .num_layers ,
10401052 num_moe_layers = _get_num_moe_layers (config .model ),
10411053 moe_layer_freq = getattr (config .model , "moe_layer_freq" , None ),
1042- mtp_num_layers = getattr ( config . model , " mtp_num_layers" , None ) ,
1054+ mtp_num_layers = mtp_num_layers ,
10431055 pg_collection = pg_collection ,
10441056 )
10451057 if getattr (config .model , "mtp_num_layers" , None ) is not None :
0 commit comments