feat(recipe): add DSv3 256-GPU GB300 MXFP8 deterministic perf recipe (#6272)

Signed-off-by: Zhiyu Li <zhiyul@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Author: ZhiyuLi-Nvidia

src/megatron/bridge/perf_recipes/deepseek/__init__.py

@@ -35,6 +35,7 @@
     deepseek_v3_pretrain_256gpu_gb300_bf16_config,
     deepseek_v3_pretrain_256gpu_gb300_fp8cs_config,
     deepseek_v3_pretrain_256gpu_gb300_fp8mx_config,
+    deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config,
     deepseek_v3_pretrain_256gpu_gb300_fp8mx_hsdp_config,
     deepseek_v3_pretrain_256gpu_gb300_fp8mx_large_scale_config,
     deepseek_v3_pretrain_256gpu_gb300_fp8mx_partial_cg_dev_config,

src/megatron/bridge/perf_recipes/deepseek/gb300/deepseek_v3.py

@@ -26,6 +26,7 @@
     set_deepseek_v3_pipeline_model_parallel_layout,
 )
 from megatron.bridge.perf_recipes.environment import COMMON_PERF_ENV_VARS
+from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
 
 
 def _build_deepseek_v3_gb300_bf16() -> ConfigContainer:
@@ -197,6 +198,30 @@ def deepseek_v3_pretrain_256gpu_gb300_fp8mx_config() -> ConfigContainer:
     return cfg
 
 
+def deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config() -> ConfigContainer:
+    """DeepSeek V3 pretrain: 256× GB300, MXFP8, deterministic.
+
+    Wraps :func:`deepseek_v3_pretrain_256gpu_gb300_fp8mx_config` and applies
+    :func:`~megatron.bridge.recipes.utils.determinism_utils.apply_determinism_overrides`.
+    """
+    cfg = deepseek_v3_pretrain_256gpu_gb300_fp8mx_config()
+    # TODO: revert back to NCCL EP once the NCCL EP issue is fixed.
+    cfg.model.moe_flex_dispatcher_backend = "hybridep"
+    cfg.model.moe_use_grouped_tensor = False
+    cfg.env_vars.pop("NCCL_EP_HT_EM_PULL_PUSH")
+    cfg.env_vars.update(
+        {
+            # HybridEP topology for the target system.
+            "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 32,
+            "NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
+            "NVLINK_DOMAIN_SIZE": 72,
+            "USE_MNNVL": 1,
+        }
+    )
+    apply_determinism_overrides(cfg)
+    return cfg
+
+
 def _build_deepseek_v3_gb300_nvfp4() -> ConfigContainer:
     """Shared HybridEP NVFP4 base for the DeepSeek V3 256-GPU GB300 recipe and its VR200 alias."""
     cfg = deepseek_v3_pretrain_config()

tests/unit_tests/recipes/test_gb300_ncclep_perf_recipes.py

@@ -17,6 +17,7 @@
 import importlib
 
 import pytest
+from megatron.training.determinism import DETERMINISM_ENV_VAR_DEFAULTS
 
 from megatron.bridge.perf_recipes.environment import HYBRID_EP_ENV_NAMES
 from tests.unit_tests.recipes.recipe_test_utils import patch_recipe_construction_dependencies
@@ -25,6 +26,7 @@
 pytestmark = pytest.mark.unit
 
 _NCCL_EP_ENV = {"NCCL_EP_HT_EM_PULL_PUSH": 1}
+_SSM_DETERMINISM_ENV = {"MAMBA_DETERMINISTIC": "1", "CAUSAL_CONV1D_DETERMINISTIC": "1"}
 
 # (module under megatron.bridge.perf_recipes, recipe factory) for the GB300 recipes defaulted to NCCL EP.
 _GB300_NCCLEP_RECIPES = (
@@ -131,3 +133,22 @@ def test_vr200_aliases_do_not_inherit_nccl_ep(module_name: str, factory_name: st
         assert cfg.env_vars.keys() >= HYBRID_EP_ENV_NAMES
     else:
         assert cfg.model.moe_token_dispatcher_type == "alltoall"
+
+
+def test_deepseek_v3_gb300_fp8mx_deterministic_is_base_plus_determinism_overrides() -> None:
+    module = "deepseek.gb300.deepseek_v3"
+    base = _build(module, "deepseek_v3_pretrain_256gpu_gb300_fp8mx_config")
+    cfg = _build(module, "deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config")
+
+    assert cfg.model.deterministic_mode is True
+    assert cfg.model.cross_entropy_loss_fusion is False
+    assert cfg.model.moe_router_aux_loss_fusion is False
+    assert cfg.model.moe_router_fusion == base.model.moe_router_fusion
+    # HybridEP with the legacy grouped MLP until the NCCL EP path is deterministic.
+    assert cfg.model.moe_flex_dispatcher_backend == "hybridep"
+    assert cfg.model.moe_use_grouped_tensor is False
+    assert cfg.env_vars.keys() >= HYBRID_EP_ENV_NAMES
+    assert cfg.env_vars.keys().isdisjoint(_NCCL_EP_ENV)
+    expected_env = {name: value for name, value in base.env_vars.items() if name not in _NCCL_EP_ENV}
+    expected_env.update({name: cfg.env_vars[name] for name in HYBRID_EP_ENV_NAMES})
+    assert cfg.env_vars == {**expected_env, **DETERMINISM_ENV_VAR_DEFAULTS, **_SSM_DETERMINISM_ENV}