Skip to content

Commit 218e24f

Browse files
fix(recipe): write determinism env vars inline in H100 and perf recipes
The recipe-environment tests parse each builder's env_vars dict with ast and only accept the COMMON_*_ENV_VARS spread plus literal entries. Spreading DETERMINISM_ENV_VARS in the Moonlight H100 and DSv3 GB300 perf recipes failed test_explicit_recipe_environment_invariants and test_every_flat_recipe_builder_declares_its_environment_inline. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Signed-off-by: Zhiyu Li <zhiyul@nvidia.com>
1 parent ca1ceb2 commit 218e24f

2 files changed

Lines changed: 15 additions & 4 deletions

File tree

‎src/megatron/bridge/perf_recipes/deepseek/gb300/deepseek_v3.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
)
2828
from megatron.bridge.perf_recipes.environment import COMMON_PERF_ENV_VARS
2929
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
30-
from megatron.bridge.recipes.utils.environment_utils import DETERMINISM_ENV_VARS
3130

3231

3332
def _build_deepseek_v3_gb300_bf16() -> ConfigContainer:
@@ -213,7 +212,13 @@ def deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config() -> ConfigCont
213212
# Keep process settings next to the recipe so users can see the exact benchmark environment.
214213
cfg.env_vars = {
215214
**COMMON_PERF_ENV_VARS,
216-
**DETERMINISM_ENV_VARS,
215+
# ---- Determinism (same values as apply_determinism_overrides) ----
216+
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
217+
"NCCL_ALGO": "Ring",
218+
"NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
219+
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
220+
"MAMBA_DETERMINISTIC": "1",
221+
"CAUSAL_CONV1D_DETERMINISTIC": "1",
217222
# ---- Others ----
218223
# CUDA stream scheduling for this model and parallel layout.
219224
"CUDA_DEVICE_MAX_CONNECTIONS": 32,

‎src/megatron/bridge/recipes/moonlight/h100/moonlight_16b.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from megatron.bridge.recipes.common import _peft_common, _pretrain_common, _sft_common
2222
from megatron.bridge.recipes.utils.dataset_utils import default_peft_config
2323
from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
24-
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
24+
from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
2525
from megatron.bridge.training.comm_overlap import CommOverlapConfig
2626
from megatron.bridge.training.config import ConfigContainer
2727
from megatron.bridge.training.mixed_precision import MixedPrecisionConfig
@@ -403,7 +403,13 @@ def moonlight_16b_pretrain_16gpu_h100_bf16_deterministic_config() -> ConfigConta
403403
# Keep the complete process environment visible on the recipe.
404404
cfg.env_vars = {
405405
**COMMON_RECIPE_ENV_VARS,
406-
**DETERMINISM_ENV_VARS,
406+
# Determinism (same values as apply_determinism_overrides).
407+
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
408+
"NCCL_ALGO": "Ring",
409+
"NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
410+
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
411+
"MAMBA_DETERMINISTIC": "1",
412+
"CAUSAL_CONV1D_DETERMINISTIC": "1",
407413
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
408414
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 8,
409415
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,

0 commit comments

Comments
 (0)