Skip to content

Commit f24809d

Browse files
committed
Merge branch 'fix/moe-mtp-metric-layer-count' of https://github.com/odedovadia/Megatron-Bridge into fix/moe-mtp-metric-layer-count
Signed-off-by: Oded Ovadia <oded7272@gmail.com> # Conflicts: # src/megatron/bridge/training/utils/train_utils.py # tests/unit_tests/training/utils/test_train_utils.py
2 parents b30c61c + a3f1f25 commit f24809d

1 file changed

Lines changed: 13 additions & 1 deletion

File tree

‎src/megatron/bridge/training/utils/train_utils.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import torch.nn as nn
2727
from megatron.core.models.hybrid.hybrid_layer_allocation import Symbols, parse_hybrid_pattern
2828
from megatron.core.num_microbatches_calculator import get_num_microbatches
29+
from megatron.core.models.hybrid.hybrid_layer_allocation import Symbols
2930
from megatron.core.tensor_parallel import param_is_not_tensor_parallel_duplicate
3031
from megatron.core.transformer.module import MegatronModule
3132
from megatron.core.transformer.moe.moe_utils import track_moe_metrics
@@ -1023,6 +1024,17 @@ def training_log(
10231024
if getattr(config.model, "moe_z_loss_coeff", None) is not None:
10241025
track_names.append("z_loss")
10251026

1027+
mtp_num_layers = getattr(config.model, "mtp_num_layers", None)
1028+
if getattr(config.model, "is_hybrid_model", False):
1029+
# MCore expects main and MTP MoE layer counts separately.
1030+
pattern = getattr(config.model, "hybrid_layer_pattern", "")
1031+
main_pattern, mtp_separator, mtp_patterns = pattern.partition(Symbols.MTP_SEPARATOR)
1032+
layers = main_pattern.count(Symbols.MOE)
1033+
if mtp_separator:
1034+
mtp_num_layers = mtp_patterns.count(Symbols.MOE)
1035+
else:
1036+
layers = getattr(config.model, "num_layers", None)
1037+
10261038
# Wrap the TB writer so MoE/MTP metrics also reach MLFlow / Comet (issue #2989).
10271039
# No-op when neither logger is configured: the original writer is returned as-is.
10281040
moe_metric_writer = _build_moe_metric_writer(writer, comet_logger, mlflow_logger)
@@ -1039,7 +1051,7 @@ def training_log(
10391051
num_layers=config.model.num_layers,
10401052
num_moe_layers=_get_num_moe_layers(config.model),
10411053
moe_layer_freq=getattr(config.model, "moe_layer_freq", None),
1042-
mtp_num_layers=getattr(config.model, "mtp_num_layers", None),
1054+
mtp_num_layers=mtp_num_layers,
10431055
pg_collection=pg_collection,
10441056
)
10451057
if getattr(config.model, "mtp_num_layers", None) is not None:

0 commit comments

Comments
 (0)