test(recipe): allow DETERMINISM_ENV_VARS as a shared recipe env constant

Restore the DETERMINISM_ENV_VARS spread in the Moonlight H100 and DSv3 GB300
recipes. The two recipe-environment tests now accept that spread next to the
common-default constants and merge its values when checking invariants, and
docs/recipe-usage.md documents it as a third shared constant.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Signed-off-by: Zhiyu Li <zhiyul@nvidia.com>

Author: ZhiyuLi-Nvidia

docs/recipe-usage.md

@@ -326,6 +326,8 @@ Flat performance and hardware-specific model recipes keep their baseline environ
 
 Every flat performance recipe builder assigns `cfg.env_vars` directly before returning its config. `COMMON_PERF_ENV_VARS` contains only `TORCH_NCCL_HIGH_PRIORITY=1`, the one value shared by every workload; all GPU-, topology-, precision-, CUDA graph-, and model-specific values are written inline with comments in the owning recipe. This is intentionally verbose so a recipe user can see the exact benchmark environment without following a decorator or global derivation rules.
 
+Deterministic recipes additionally spread `DETERMINISM_ENV_VARS` (from `megatron.bridge.recipes.utils.environment_utils`), a third shared constant, next to the two common-default ones, holding the process settings required for bit-exact reproducibility, such as `CUBLAS_WORKSPACE_CONFIG` and `NCCL_ALGO`. It is the only other constant a recipe may spread, and it is spread after `apply_determinism_overrides` because the recipe reassigns `cfg.env_vars`.
+
 Hardware recipe builders follow the same explicit pattern. `COMMON_RECIPE_ENV_VARS` contains only the seven settings that are identical across every supported recipe; each of the 245 supported builders assigns `cfg.env_vars` directly before returning and writes any HybridEP or model-specific additions inline. There is no recipe decorator or generic launcher-side environment recomposition. Because the compatibility catalog currently reuses H100 builders on other targets, the recipe entry point adapts HybridEP topology after overrides and preserves legacy coupling for an explicit argparse `--nccl_ub`; explicit `env_vars` remain final. The nemo-ci executors under `scripts/performance` do not provide fallback values for recipe-owned process settings.
 
 Cluster fabric and orchestration settings do not belong in recipes. Keep credentials, cache/offline policy, scheduler metadata, W&B state, and cluster-specific NCCL network tuning in the launcher or executor. In particular, settings such as `NCCL_NET_GDR_LEVEL`, `NCCL_NET_GDR_C2C`, and `NCCL_IB_QPS_PER_CONNECTION` must remain deployment-owned.

src/megatron/bridge/perf_recipes/deepseek/gb300/deepseek_v3.py

@@ -27,6 +27,7 @@
 )
 from megatron.bridge.perf_recipes.environment import COMMON_PERF_ENV_VARS
 from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
+from megatron.bridge.recipes.utils.environment_utils import DETERMINISM_ENV_VARS
 
 
 def _build_deepseek_v3_gb300_bf16() -> ConfigContainer:
@@ -212,13 +213,7 @@ def deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config() -> ConfigCont
     # Keep process settings next to the recipe so users can see the exact benchmark environment.
     cfg.env_vars = {
         **COMMON_PERF_ENV_VARS,
-        # ---- Determinism (same values as apply_determinism_overrides) ----
-        "CUBLAS_WORKSPACE_CONFIG": ":4096:8",
-        "NCCL_ALGO": "Ring",
-        "NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
-        "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
-        "MAMBA_DETERMINISTIC": "1",
-        "CAUSAL_CONV1D_DETERMINISTIC": "1",
+        **DETERMINISM_ENV_VARS,
         # ---- Others ----
         # CUDA stream scheduling for this model and parallel layout.
         "CUDA_DEVICE_MAX_CONNECTIONS": 32,

src/megatron/bridge/recipes/moonlight/h100/moonlight_16b.py

@@ -21,7 +21,7 @@
 from megatron.bridge.recipes.common import _peft_common, _pretrain_common, _sft_common
 from megatron.bridge.recipes.utils.dataset_utils import default_peft_config
 from megatron.bridge.recipes.utils.determinism_utils import apply_determinism_overrides
-from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
+from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
 from megatron.bridge.training.comm_overlap import CommOverlapConfig
 from megatron.bridge.training.config import ConfigContainer
 from megatron.bridge.training.mixed_precision import MixedPrecisionConfig
@@ -403,13 +403,7 @@ def moonlight_16b_pretrain_16gpu_h100_bf16_deterministic_config() -> ConfigConta
     # Keep the complete process environment visible on the recipe.
     cfg.env_vars = {
         **COMMON_RECIPE_ENV_VARS,
-        # Determinism (same values as apply_determinism_overrides).
-        "CUBLAS_WORKSPACE_CONFIG": ":4096:8",
-        "NCCL_ALGO": "Ring",
-        "NCCL_EP_HT_EM_AG_SCAN_MODE": "1",
-        "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
-        "MAMBA_DETERMINISTIC": "1",
-        "CAUSAL_CONV1D_DETERMINISTIC": "1",
+        **DETERMINISM_ENV_VARS,
         "CUDA_DEVICE_MAX_CONNECTIONS": 32,
         "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 8,
         "NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,

tests/unit_tests/recipes/test_perf_recipe_environment.py

@@ -23,6 +23,7 @@
 
 from megatron.bridge.perf_recipes._common import _benchmark_common
 from megatron.bridge.perf_recipes.environment import COMMON_PERF_ENV_VARS
+from megatron.bridge.recipes.utils.environment_utils import DETERMINISM_ENV_VARS
 
 
 _CANONICAL_RECIPE_NAME = re.compile(
@@ -101,8 +102,11 @@ def _explicit_environment(path: Path, function_name: str) -> dict[str, str | int
     common_expansions = 0
     for key, value in zip(mapping.keys, mapping.values):
         if key is None:
-            assert isinstance(value, ast.Name) and value.id == "COMMON_PERF_ENV_VARS"
-            common_expansions += 1
+            assert isinstance(value, ast.Name) and value.id in {"COMMON_PERF_ENV_VARS", "DETERMINISM_ENV_VARS"}
+            if value.id == "COMMON_PERF_ENV_VARS":
+                common_expansions += 1
+            else:
+                result.update(DETERMINISM_ENV_VARS)
             continue
         if isinstance(value, ast.Name):
             assert value.id in local_constants

tests/unit_tests/recipes/test_recipe_environment_defaults.py

@@ -20,7 +20,7 @@
 
 import pytest
 
-from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS
+from megatron.bridge.recipes.utils.environment_utils import COMMON_RECIPE_ENV_VARS, DETERMINISM_ENV_VARS
 
 
 _CANONICAL_RECIPE_NAME = re.compile(
@@ -67,9 +67,12 @@ def _explicit_environment(path: Path, function_name: str) -> dict[str, str | int
     common_expansions = 0
     for index, (key, value) in enumerate(zip(mapping.keys, mapping.values)):
         if key is None:
-            assert index == 0
-            assert isinstance(value, ast.Name) and value.id == "COMMON_RECIPE_ENV_VARS"
-            common_expansions += 1
+            assert isinstance(value, ast.Name) and value.id in {"COMMON_RECIPE_ENV_VARS", "DETERMINISM_ENV_VARS"}
+            if value.id == "COMMON_RECIPE_ENV_VARS":
+                assert index == 0
+                common_expansions += 1
+            else:
+                result.update(DETERMINISM_ENV_VARS)
             continue
         result[ast.literal_eval(key)] = ast.literal_eval(value)
     assert common_expansions == 1