Skip to content

Commit 3173b86

Browse files
authored
[recipe] fix: Correct Ministral 3 SFT learning rates (#5092)
Signed-off-by: Yu Yao <yaoyu.094@gmail.com>
1 parent 3e0dfe1 commit 3173b86

2 files changed

Lines changed: 19 additions & 6 deletions

File tree

‎src/megatron/bridge/recipes/ministral3/h100/ministral3.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,8 @@ def ministral3_3b_sft_1gpu_h100_bf16_config() -> ConfigContainer:
9494
opt_cfg, scheduler_cfg = distributed_fused_adam_with_cosine_annealing(
9595
lr_warmup_iters=10,
9696
lr_decay_iters=50,
97-
max_lr=0.00005,
98-
min_lr=0.000005,
97+
max_lr=0.000005,
98+
min_lr=0.0000005,
9999
)
100100
cfg.optimizer = opt_cfg
101101
cfg.scheduler = scheduler_cfg
@@ -209,8 +209,8 @@ def ministral3_8b_sft_2gpu_h100_bf16_config() -> ConfigContainer:
209209
opt_cfg, scheduler_cfg = distributed_fused_adam_with_cosine_annealing(
210210
lr_warmup_iters=10,
211211
lr_decay_iters=50,
212-
max_lr=0.00005,
213-
min_lr=0.000005,
212+
max_lr=0.000005,
213+
min_lr=0.0000005,
214214
)
215215
cfg.optimizer = opt_cfg
216216
cfg.scheduler = scheduler_cfg
@@ -324,8 +324,8 @@ def ministral3_14b_sft_4gpu_h100_bf16_config() -> ConfigContainer:
324324
opt_cfg, scheduler_cfg = distributed_fused_adam_with_cosine_annealing(
325325
lr_warmup_iters=10,
326326
lr_decay_iters=50,
327-
max_lr=0.00005,
328-
min_lr=0.000005,
327+
max_lr=0.000005,
328+
min_lr=0.0000005,
329329
)
330330
cfg.optimizer = opt_cfg
331331
cfg.scheduler = scheduler_cfg

‎tests/unit_tests/recipes/test_ministral3_recipes.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,19 @@ def test_each_ministral3_sft_recipe_builds_config(recipe_func: Callable, monkeyp
125125
assert cfg.peft is None
126126

127127

128+
@pytest.mark.parametrize("recipe_func", _MINISTRAL3_SFT_FUNCS)
129+
def test_each_ministral3_sft_recipe_uses_recommended_learning_rate(
130+
recipe_func: Callable, monkeypatch: pytest.MonkeyPatch
131+
):
132+
"""Test that each Ministral3 full-SFT recipe uses its recommended learning-rate range."""
133+
patch_recipe_module_global(monkeypatch, _ministral3_module, "AutoBridge", _FakeAutoBridge)
134+
135+
cfg = recipe_func()
136+
137+
assert cfg.optimizer.lr == pytest.approx(5e-6)
138+
assert cfg.optimizer.min_lr == pytest.approx(5e-7)
139+
140+
128141
@pytest.mark.parametrize("recipe_func", _MINISTRAL3_PEFT_FUNCS)
129142
def test_each_ministral3_peft_recipe_builds_config(recipe_func: Callable, monkeypatch: pytest.MonkeyPatch):
130143
"""Test that each Ministral3 PEFT recipe function builds a valid configuration."""

0 commit comments

Comments
 (0)