Skip to content

Commit 057bdbd

Browse files
ZhiyuLi-NvidiaclaudeadityavavreNVDA
authored
cp: feat(recipe): add deterministic Nemotron 3 Ultra, Qwen3.5-9B and Moonlight-16B pretrain recipes (6440) into r0.7.0 (#6497)
Signed-off-by: Zhiyu Li <zhiyul@nvidia.com> Signed-off-by: adityavavreNVDA <avavre@nvidia.com> Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: adityavavreNVDA <avavre@nvidia.com>
1 parent 39ab668 commit 057bdbd

20 files changed

Lines changed: 266 additions & 19 deletions

‎docs/recipe-usage.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,8 @@ Flat performance and hardware-specific model recipes keep their baseline environ
326326

327327
Every flat performance recipe builder assigns `cfg.env_vars` directly before returning its config. `COMMON_PERF_ENV_VARS` contains only `TORCH_NCCL_HIGH_PRIORITY=1`, the one value shared by every workload; all GPU-, topology-, precision-, CUDA graph-, and model-specific values are written inline with comments in the owning recipe. This is intentionally verbose so a recipe user can see the exact benchmark environment without following a decorator or global derivation rules.
328328

329+
Deterministic recipes additionally spread `DETERMINISM_ENV_VARS` (from `megatron.bridge.recipes.utils.environment_utils`), a third shared constant, next to the two common-default ones, holding the process settings required for bit-exact reproducibility, such as `CUBLAS_WORKSPACE_CONFIG` and `NCCL_ALGO`. It is the only other constant a recipe may spread, and it is spread after `apply_determinism_overrides` because the recipe reassigns `cfg.env_vars`.
330+
329331
Hardware recipe builders follow the same explicit pattern. `COMMON_RECIPE_ENV_VARS` contains only the seven settings that are identical across every supported recipe; each of the 245 supported builders assigns `cfg.env_vars` directly before returning and writes any HybridEP or model-specific additions inline. There is no recipe decorator or generic launcher-side environment recomposition. Because the compatibility catalog currently reuses H100 builders on other targets, the recipe entry point adapts HybridEP topology after overrides and preserves legacy coupling for an explicit argparse `--nccl_ub`; explicit `env_vars` remain final. The nemo-ci executors under `scripts/performance` do not provide fallback values for recipe-owned process settings.
330332

331333
Cluster fabric and orchestration settings do not belong in recipes. Keep credentials, cache/offline policy, scheduler metadata, W&B state, and cluster-specific NCCL network tuning in the launcher or executor. In particular, settings such as `NCCL_NET_GDR_LEVEL`, `NCCL_NET_GDR_C2C`, and `NCCL_IB_QPS_PER_CONNECTION` must remain deployment-owned.

‎src/megatron/bridge/perf_recipes/deepseek/gb300/deepseek_v3.py‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
)
2828
from megatron.bridge.perf_recipes.environment import COMMON_PERF_ENV_VARS
2929
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
30+
from megatron.bridge.recipes.utils.environment_utils import DETERMINISM_ENV_VARS
3031

3132

3233
def _build_deepseek_v3_gb300_bf16() -> ConfigContainer:
@@ -212,13 +213,7 @@ def deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config() -> ConfigCont
212213
# Keep process settings next to the recipe so users can see the exact benchmark environment.
213214
cfg.env_vars = {
214215
**COMMON_PERF_ENV_VARS,
215-
# ---- Determinism (same values as apply_determinism_overrides) ----
216-
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
217-
"NCCL_ALGO": "Ring",
218-
"NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
219-
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
220-
"MAMBA_DETERMINISTIC": "1",
221-
"CAUSAL_CONV1D_DETERMINISTIC": "1",
216+
**DETERMINISM_ENV_VARS,
222217
# ---- Others ----
223218
# CUDA stream scheduling for this model and parallel layout.
224219
"CUDA_DEVICE_MAX_CONNECTIONS": 32,

‎src/megatron/bridge/recipes/moonlight/h100/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
"moonlight_16b_peft_2gpu_h100_bf16_config",
2020
"moonlight_16b_peft_4gpu_h100_bf16_config",
2121
"moonlight_16b_pretrain_16gpu_h100_bf16_config",
22+
"moonlight_16b_pretrain_16gpu_h100_bf16_deterministic_config",
2223
"moonlight_16b_pretrain_8gpu_h100_bf16_config",
2324
"moonlight_16b_sft_8gpu_h100_bf16_8k_config",
2425
"moonlight_16b_sft_8gpu_h100_bf16_config",

‎src/megatron/bridge/recipes/moonlight/h100/moonlight_16b.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
from megatron.bridge.peft.base import PEFT
2121
from megatron.bridge.recipes.common import _peft_common, _pretrain_common, _sft_common
2222
from megatron.bridge.recipes.utils.dataset_utils import default_peft_config
23-
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
23+
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
24+
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
2425
from megatron.bridge.training.comm_overlap import CommOverlapConfig
2526
from megatron.bridge.training.config import ConfigContainer
2627
from megatron.bridge.training.mixed_precision import MixedPrecisionConfig
@@ -391,6 +392,29 @@ def moonlight_16b_pretrain_16gpu_h100_bf16_config() -> ConfigContainer:
391392
return cfg
392393

393394

395+
def moonlight_16b_pretrain_16gpu_h100_bf16_deterministic_config() -> ConfigContainer:
396+
"""Return a deterministic bounded-convergence pre-training config for Moonlight-16B.
397+
398+
Wraps :func:`moonlight_16b_pretrain_16gpu_h100_bf16_config` and applies
399+
:func:`~megatron.bridge.recipes.utils.determinism_utils.apply_determinism_overrides`.
400+
"""
401+
cfg = moonlight_16b_pretrain_16gpu_h100_bf16_config()
402+
apply_determinism_overrides(cfg)
403+
# Keep the complete process environment visible on the recipe.
404+
cfg.env_vars = {
405+
**COMMON_RECIPE_ENV_VARS,
406+
**DETERMINISM_ENV_VARS,
407+
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
408+
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 8,
409+
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
410+
"NVLINK_DOMAIN_SIZE": 8,
411+
"NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
412+
"NVTE_FWD_LAYERNORM_SM_MARGIN": 20,
413+
"USE_MNNVL": 0,
414+
}
415+
return cfg
416+
417+
394418
# =============================================================================
395419
# SFT Config
396420
# =============================================================================
@@ -912,6 +936,7 @@ def moonlight_16b_peft_4gpu_h100_bf16_config(
912936
"moonlight_16b_peft_2gpu_h100_bf16_config",
913937
"moonlight_16b_peft_4gpu_h100_bf16_config",
914938
"moonlight_16b_pretrain_16gpu_h100_bf16_config",
939+
"moonlight_16b_pretrain_16gpu_h100_bf16_deterministic_config",
915940
"moonlight_16b_pretrain_8gpu_h100_bf16_config",
916941
"moonlight_16b_sft_8gpu_h100_bf16_8k_config",
917942
"moonlight_16b_sft_8gpu_h100_bf16_config",

‎src/megatron/bridge/recipes/nemotronh/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@
1919
nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config,
2020
nemotron_3_nano_gb200_pretrain_config,
2121
nemotron_3_nano_pretrain_8gpu_gb200_bf16_config,
22+
nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config,
2223
nemotron_3_super_pretrain_64gpu_gb200_bf16_config,
2324
nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config,
25+
nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config,
2426
)
2527
from megatron.bridge.recipes.nemotronh.nemotron_3_nano import (
2628
nemotron_3_5_lightning_peft_config,
@@ -65,6 +67,7 @@
6567
"nemotron_3_nano_peft_config",
6668
"nemotron_3_nano_gb200_pretrain_config",
6769
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_config",
70+
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config",
6871
# Nemotron 3 Nano 4B model
6972
"nemotron_3_nano_4b_pretrain_config",
7073
"nemotron_3_nano_4b_sft_config",
@@ -77,6 +80,7 @@
7780
"nemotron_3_super_peft_config",
7881
# Nemotron 3 Ultra models
7982
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config",
83+
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config",
8084
"nemotron_3_ultra_pretrain_config",
8185
"nemotron_3_ultra_sft_openmathinstruct2_packed_config",
8286
"nemotron_3_ultra_peft_openmathinstruct2_packed_config",

‎src/megatron/bridge/recipes/nemotronh/gb200/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@
1818
nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config,
1919
nemotron_3_nano_gb200_pretrain_config,
2020
nemotron_3_nano_pretrain_8gpu_gb200_bf16_config,
21+
nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config,
2122
)
2223
from megatron.bridge.recipes.nemotronh.gb200.nemotron_3_super import (
2324
nemotron_3_super_pretrain_64gpu_gb200_bf16_config,
2425
)
2526
from megatron.bridge.recipes.nemotronh.gb200.nemotron_3_ultra import (
2627
nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config,
28+
nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config,
2729
)
2830

2931

@@ -33,6 +35,8 @@
3335
"nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config",
3436
"nemotron_3_nano_gb200_pretrain_config",
3537
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_config",
38+
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config",
3639
"nemotron_3_super_pretrain_64gpu_gb200_bf16_config",
3740
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config",
41+
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config",
3842
]

‎src/megatron/bridge/recipes/nemotronh/gb200/nemotron_3_nano.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
nemotron_3_5_lightning_pretrain_config,
2323
nemotron_3_5_lightning_sft_openmathinstruct2_packed_config,
2424
)
25-
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
25+
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
26+
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
2627
from megatron.bridge.training.comm_overlap import CommOverlapConfig
2728
from megatron.bridge.training.config import ConfigContainer
2829
from megatron.bridge.training.mixed_precision import get_mixed_precision_config
@@ -157,6 +158,31 @@ def nemotron_3_nano_pretrain_8gpu_gb200_bf16_config() -> ConfigContainer:
157158
return cfg
158159

159160

161+
def nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config() -> ConfigContainer:
162+
"""Return a deterministic Nemotron 3 Nano BF16 pretraining config for eight GB200 GPUs.
163+
164+
Wraps :func:`nemotron_3_nano_pretrain_8gpu_gb200_bf16_config` and applies
165+
:func:`~megatron.bridge.recipes.utils.determinism_utils.apply_determinism_overrides`.
166+
"""
167+
cfg = nemotron_3_nano_pretrain_8gpu_gb200_bf16_config()
168+
apply_determinism_overrides(cfg)
169+
# Keep the exact process environment visible on the recipe.
170+
cfg.env_vars = {
171+
**COMMON_RECIPE_ENV_VARS,
172+
**DETERMINISM_ENV_VARS,
173+
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
174+
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 8,
175+
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
176+
"NVLINK_DOMAIN_SIZE": 72,
177+
"USE_MNNVL": 1,
178+
"NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
179+
"NVTE_FWD_LAYERNORM_SM_MARGIN": 20,
180+
"NVTE_NORM_BWD_USE_CUDNN": 1,
181+
"NVTE_NORM_FWD_USE_CUDNN": 1,
182+
}
183+
return cfg
184+
185+
160186
def nemotron_3_5_lightning_pretrain_8k_config() -> ConfigContainer:
161187
"""Return the Nemotron 3.5 Lightning BF16 8K pretraining config."""
162188
cfg = nemotron_3_5_lightning_pretrain_config()
@@ -250,4 +276,5 @@ def nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config() -> ConfigCo
250276
"nemotron_3_5_lightning_sft_openmathinstruct2_packed_tp1_config",
251277
"nemotron_3_nano_gb200_pretrain_config",
252278
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_config",
279+
"nemotron_3_nano_pretrain_8gpu_gb200_bf16_deterministic_config",
253280
]

‎src/megatron/bridge/recipes/nemotronh/gb200/nemotron_3_ultra.py‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
NEMOTRON_3_ULTRA_PRETRAIN_SEQ_LENGTH,
2121
_nemotron_3_ultra_large_scale_bf16_config,
2222
)
23-
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
23+
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
24+
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
2425
from megatron.bridge.training.config import ConfigContainer
2526

2627

@@ -100,4 +101,36 @@ def nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config() -> ConfigContainer
100101
return _nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config()
101102

102103

103-
__all__ = ["nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config"]
104+
def nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config() -> ConfigContainer:
105+
"""Return a deterministic Nemotron 3 Ultra BF16 pretrain config for 256 GB200 GPUs.
106+
107+
Wraps :func:`nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config` and applies
108+
:func:`~megatron.bridge.recipes.utils.determinism_utils.apply_determinism_overrides`
109+
and spreads ``DETERMINISM_ENV_VARS`` into the recipe's static process environment.
110+
111+
Returns:
112+
GB200 BF16 deterministic pretraining configuration.
113+
"""
114+
cfg = nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config()
115+
apply_determinism_overrides(cfg)
116+
# Keep the exact process environment visible on the recipe.
117+
cfg.env_vars = {
118+
**COMMON_RECIPE_ENV_VARS,
119+
**DETERMINISM_ENV_VARS,
120+
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
121+
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 16,
122+
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
123+
"NVLINK_DOMAIN_SIZE": 72,
124+
"USE_MNNVL": 1,
125+
"NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
126+
"NVTE_CPU_OFFLOAD_V1": 1,
127+
"NVTE_CUTEDSL_FUSED_GROUPED_MLP": 1,
128+
"NVTE_FWD_LAYERNORM_SM_MARGIN": 20,
129+
}
130+
return cfg
131+
132+
133+
__all__ = [
134+
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_config",
135+
"nemotron_3_ultra_pretrain_256gpu_gb200_bf16_ep16_deterministic_config",
136+
]

‎src/megatron/bridge/recipes/qwen/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
# Qwen3.5 GB200 models
2121
from .gb200.qwen35 import (
2222
qwen35_text_9b_pretrain_8gpu_gb200_bf16_config,
23+
qwen35_text_9b_pretrain_8gpu_gb200_bf16_deterministic_config,
2324
qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config,
2425
qwen35_text_35b_a3b_sft_8gpu_gb200_bf16_dynamic_cp_config,
2526
qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config,
@@ -175,6 +176,7 @@
175176
# Qwen3.5 text models
176177
"qwen35_text_9b_pretrain_config",
177178
"qwen35_text_9b_pretrain_8gpu_gb200_bf16_config",
179+
"qwen35_text_9b_pretrain_8gpu_gb200_bf16_deterministic_config",
178180
"qwen35_text_35b_a3b_pretrain_config",
179181
"qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config",
180182
"qwen35_text_35b_a3b_sft_8gpu_gb200_bf16_dynamic_cp_config",

‎src/megatron/bridge/recipes/qwen/gb200/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
)
1818
from megatron.bridge.recipes.qwen.gb200.qwen35 import (
1919
qwen35_text_9b_pretrain_8gpu_gb200_bf16_config,
20+
qwen35_text_9b_pretrain_8gpu_gb200_bf16_deterministic_config,
2021
qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config,
2122
qwen35_text_35b_a3b_sft_8gpu_gb200_bf16_dynamic_cp_config,
2223
qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config,
@@ -26,6 +27,7 @@
2627
__all__ = [
2728
"qwen3_30b_a3b_pretrain_8gpu_gb200_fp8mx_config",
2829
"qwen35_text_9b_pretrain_8gpu_gb200_bf16_config",
30+
"qwen35_text_9b_pretrain_8gpu_gb200_bf16_deterministic_config",
2931
"qwen35_text_35b_a3b_pretrain_8gpu_gb200_bf16_config",
3032
"qwen35_text_35b_a3b_sft_8gpu_gb200_bf16_dynamic_cp_config",
3133
"qwen35_text_35b_a3b_sft_long_context_16gpu_gb200_fp8mx_config",

0 commit comments

Comments
 (0)