Skip to content

Commit 63fe8d6

Browse files
authored
Merge branch 'main' into rob-luke/training/mlflow-status
2 parents 05514ac + 2e3c340 commit 63fe8d6

17 files changed

Lines changed: 103 additions & 161 deletions

‎.main.commit‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
e41b37002cd8df1cd97c93e3e0876cf0850f72f8
1+
b2a8ec7458e90216e3a8d147d8ddf81cfb895f7c

‎3rdparty/Megatron-LM‎

Submodule Megatron-LM updated 148 files

‎src/megatron/bridge/models/glm/glm45_bridge.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,11 @@
2323
from megatron.bridge.models.conversion.model_bridge import MegatronModelBridge
2424
from megatron.bridge.models.conversion.param_mapping import (
2525
AutoMapping,
26+
FusedExpertMapping,
27+
FusedGatedExpertMapping,
2628
GatedMLPMapping,
2729
QKVMapping,
2830
)
29-
from megatron.bridge.models.glm.glm_moe_mappings import (
30-
GLMExpertDownProjMapping,
31-
GLMExpertGateUpProjMapping,
32-
)
3331
from megatron.bridge.models.gpt_provider import GPTModelProvider
3432
from megatron.bridge.models.hf_pretrained.causal_lm import PreTrainedCausalLM
3533

@@ -168,11 +166,11 @@ def mapping_registry(self) -> MegatronMappingRegistry:
168166
if use_fused_experts:
169167
mapping_list.extend(
170168
[
171-
GLMExpertGateUpProjMapping(
169+
FusedGatedExpertMapping(
172170
megatron_param="decoder.layers.*.mlp.experts.linear_fc1.weight*",
173171
hf_param=f"model.layers.*.mlp.experts.gate_up_proj{gate_up_suffix}",
174172
),
175-
GLMExpertDownProjMapping(
173+
FusedExpertMapping(
176174
megatron_param="decoder.layers.*.mlp.experts.linear_fc2.weight*",
177175
hf_param=f"model.layers.*.mlp.experts.down_proj{down_suffix}",
178176
),
@@ -240,7 +238,7 @@ def mapping_registry(self) -> MegatronMappingRegistry:
240238
if use_fused_experts:
241239
mapping_list.extend(
242240
[
243-
GLMExpertGateUpProjMapping(
241+
FusedGatedExpertMapping(
244242
megatron_param=(
245243
f"mtp.layers.{mtp_layer}.{layer_prefix}.mlp.experts.linear_fc1.weight*"
246244
),
@@ -249,7 +247,7 @@ def mapping_registry(self) -> MegatronMappingRegistry:
249247
f"{gate_up_suffix}"
250248
),
251249
),
252-
GLMExpertDownProjMapping(
250+
FusedExpertMapping(
253251
megatron_param=(
254252
f"mtp.layers.{mtp_layer}.{layer_prefix}.mlp.experts.linear_fc2.weight*"
255253
),

‎src/megatron/bridge/models/glm/glm_moe_mappings.py‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎src/megatron/bridge/models/glm_vl/glm_45v_bridge.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,13 @@
1919
from megatron.bridge.models.conversion.model_bridge import MegatronModelBridge
2020
from megatron.bridge.models.conversion.param_mapping import (
2121
AutoMapping,
22+
FusedExpertMapping,
23+
FusedGatedExpertMapping,
2224
GatedMLPMapping,
2325
QKVMapping,
2426
ReplicatedMapping,
2527
)
2628
from megatron.bridge.models.conversion.transformers_compat import rope_theta_from_hf
27-
from megatron.bridge.models.glm.glm_moe_mappings import (
28-
GLMExpertDownProjMapping,
29-
GLMExpertGateUpProjMapping,
30-
)
3129
from megatron.bridge.models.glm_vl.glm_45v_provider import GLM45VModelProvider
3230
from megatron.bridge.models.glm_vl.modeling_glm_45v import GLM45VModel
3331
from megatron.bridge.models.hf_pretrained.vlm import PreTrainedVLM
@@ -178,11 +176,11 @@ def mapping_registry(self) -> MegatronMappingRegistry:
178176
if use_fused_experts:
179177
mapping_list.extend(
180178
[
181-
GLMExpertGateUpProjMapping(
179+
FusedGatedExpertMapping(
182180
megatron_param="language_model.decoder.layers.*.mlp.experts.linear_fc1.weight*",
183181
hf_param=f"model.language_model.layers.*.mlp.experts.gate_up_proj{gate_up_suffix}",
184182
),
185-
GLMExpertDownProjMapping(
183+
FusedExpertMapping(
186184
megatron_param="language_model.decoder.layers.*.mlp.experts.linear_fc2.weight*",
187185
hf_param=f"model.language_model.layers.*.mlp.experts.down_proj{down_suffix}",
188186
),

‎src/megatron/bridge/training/checkpointing.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,6 +1215,8 @@ def wandb_finalize_fn() -> None:
12151215
)
12161216

12171217
def mlflow_finalize_fn() -> None:
1218+
if not state.cfg.logger.mlflow_log_artifacts:
1219+
return
12181220
mlflow_utils.on_save_checkpoint_success(
12191221
checkpoint_name,
12201222
save_dir,

‎src/megatron/bridge/training/config.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -712,6 +712,9 @@ class LoggerConfig(MTrainLoggerConfig):
712712
mlflow_tags: Optional[dict[str, str]] = None
713713
"""Optional tags to apply to the MLFlow run."""
714714

715+
mlflow_log_artifacts: bool = True
716+
"""Whether to upload checkpoint artifacts to MLFlow via HTTP after each save."""
717+
715718
comet_project: Optional[str] = None
716719
"""The Comet ML project name. Comet logging is disabled when this is None."""
717720

‎src/megatron/bridge/utils/slurm_utils.py‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -125,26 +125,32 @@ def _parse_slurm_nodelist(nodelist: str) -> str:
125125
- Simple list: "node001,node002" -> "node001"
126126
- Range: "node[001-004]" -> "node001"
127127
- List in brackets: "node[001,003,005]" -> "node001"
128+
- Mixed entries: "nodeA,nodeB[1-3],nodeC" -> "nodeA"
128129
129130
Args:
130131
nodelist: The SLURM nodelist string to parse.
131132
132133
Returns:
133134
The hostname of the first node in the list.
134135
"""
135-
# Handle bracket notation: "prefix[range]" or "prefix[list]"
136-
if "[" in nodelist:
137-
# Split into base and range part
138-
# e.g., "node[001-004]" -> base="node", range_part="001-004"
139-
base = nodelist.split("[")[0]
140-
range_part = nodelist.split("[")[1].split("]")[0]
141-
142-
# Handle both ranges (001-004) and lists (001,003,005)
143-
# Extract first element
144-
first_element = range_part.split(",")[0].split("-")[0]
145-
136+
# Find the first entry by scanning for a top-level comma (i.e., one
137+
# that is not inside a "[...]" group).
138+
in_bracket = False
139+
end = len(nodelist)
140+
for i, ch in enumerate(nodelist):
141+
if ch == "[":
142+
in_bracket = True
143+
elif ch == "]":
144+
in_bracket = False
145+
elif ch == "," and not in_bracket:
146+
end = i
147+
break
148+
first = nodelist[:end].strip()
149+
150+
# If the first entry itself uses bracket expansion, expand to the first node.
151+
if "[" in first:
152+
base, _, rest = first.partition("[")
153+
inside = rest.split("]")[0]
154+
first_element = inside.split(",")[0].split("-")[0]
146155
return f"{base}{first_element}"
147-
else:
148-
# Simple comma-separated list
149-
# e.g., "node001,node002,node003" -> "node001"
150-
return nodelist.split(",")[0].strip()
156+
return first

‎tests/unit_tests/models/megatron_mimo/test_compare_outputs.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,7 @@
1515

1616

1717
VERIFY_PATH = (
18-
Path(__file__).resolve().parents[4]
19-
/ "examples"
20-
/ "models"
21-
/ "megatron_mimo"
22-
/ "megatron_mimo"
23-
/ "whisper"
24-
/ "verify_whisper_conversion.py"
18+
Path(__file__).resolve().parents[4] / "examples" / "megatron_mimo" / "whisper" / "verify_whisper_conversion.py"
2519
)
2620

2721

‎tests/unit_tests/models/megatron_mimo/test_synthesize_audio_helpers.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,7 @@
1313
import pytest
1414

1515

16-
SYNTH_PATH = (
17-
Path(__file__).resolve().parents[4]
18-
/ "examples"
19-
/ "models"
20-
/ "megatron_mimo"
21-
/ "megatron_mimo"
22-
/ "synthesize_llava_pretrain_audio.py"
23-
)
16+
SYNTH_PATH = Path(__file__).resolve().parents[4] / "examples" / "megatron_mimo" / "synthesize_llava_pretrain_audio.py"
2417

2518

2619
@pytest.fixture(scope="module")

0 commit comments

Comments
 (0)