Skip to content

Commit 855f0b5

Browse files
fix(recipe): declare DSv3 GB300 deterministic recipe environment inline
Flat perf recipes must assign one inline cfg.env_vars literal (docs/recipe-usage.md, enforced by test_perf_recipe_environment.py). Replace the pop/update on the parent mapping with the full literal, grouped into determinism and other settings. The resulting environment is unchanged. Allow *_deterministic_config recipes to pin NVTE_ALLOW_NONDETERMINISTIC_ALGO=0 in the environment invariant test. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Zhiyu Li <zhiyul@nvidia.com>
1 parent 6d958fb commit 855f0b5

2 files changed

Lines changed: 36 additions & 11 deletions

File tree

‎src/megatron/bridge/perf_recipes/deepseek/gb300/deepseek_v3.py‎

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -208,17 +208,39 @@ def deepseek_v3_pretrain_256gpu_gb300_fp8mx_deterministic_config() -> ConfigCont
208208
# TODO: revert back to NCCL EP once the NCCL EP issue is fixed.
209209
cfg.model.moe_flex_dispatcher_backend = "hybridep"
210210
cfg.model.moe_use_grouped_tensor = False
211-
cfg.env_vars.pop("NCCL_EP_HT_EM_PULL_PUSH")
212-
cfg.env_vars.update(
213-
{
214-
# HybridEP topology for the target system.
215-
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 32,
216-
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
217-
"NVLINK_DOMAIN_SIZE": 72,
218-
"USE_MNNVL": 1,
219-
}
220-
)
221211
apply_determinism_overrides(cfg)
212+
# Keep process settings next to the recipe so users can see the exact benchmark environment.
213+
cfg.env_vars = {
214+
**COMMON_PERF_ENV_VARS,
215+
# ---- Determinism (same values as apply_determinism_overrides) ----
216+
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
217+
"NCCL_ALGO": "Ring",
218+
"NVTE_ALLOW_NONDETERMINISTIC_ALGO": "0",
219+
"MAMBA_DETERMINISTIC": "1",
220+
"CAUSAL_CONV1D_DETERMINISTIC": "1",
221+
# ---- Others ----
222+
# CUDA stream scheduling for this model and parallel layout.
223+
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
224+
# CUDA graph and allocator behavior for this recipe.
225+
"NCCL_GRAPH_REGISTER": 0,
226+
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True,graph_capture_record_stream_reuse:True",
227+
"TORCH_NCCL_AVOID_RECORD_STREAMS": 0,
228+
# NCCL user-buffer and launch settings.
229+
"NCCL_NVLS_ENABLE": 0,
230+
# HybridEP topology for the target system.
231+
"NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN": 32,
232+
"NUM_OF_TOKENS_PER_CHUNK_COMBINE_API": 128,
233+
"NVLINK_DOMAIN_SIZE": 72,
234+
"USE_MNNVL": 1,
235+
# Transformer Engine overlap settings for this model.
236+
"CUDNNFE_CLUSTER_OVERLAP_MARGIN": 8,
237+
"NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
238+
"NVTE_CUTEDSL_FUSED_GROUPED_MLP": 1,
239+
"NVTE_FWD_LAYERNORM_SM_MARGIN": 20,
240+
# Use cuDNN LayerNorm for this measured baseline.
241+
"NVTE_NORM_BWD_USE_CUDNN": 1,
242+
"NVTE_NORM_FWD_USE_CUDNN": 1,
243+
}
222244
return cfg
223245

224246

‎tests/unit_tests/recipes/test_perf_recipe_environment.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,10 @@ def test_explicit_environment_invariants_across_all_flat_recipes():
217217
assert environment.keys().isdisjoint(_DEEPSEEK_NON_BASELINE_ENV_NAMES)
218218
assert environment["NVTE_FWD_LAYERNORM_SM_MARGIN"] == 20
219219
assert environment["NVTE_BWD_LAYERNORM_SM_MARGIN"] == 20
220-
if path.parent.name == "gb300" and path.stem == "deepseek_v3":
220+
if function_name.endswith("_deterministic_config"):
221+
# Bit-exact recipes pin deterministic TE kernels.
222+
assert environment["NVTE_ALLOW_NONDETERMINISTIC_ALGO"] == "0"
223+
elif path.parent.name == "gb300" and path.stem == "deepseek_v3":
221224
# The DeepSeek V3 GB300 recipes run the TE op-fuser grouped MLP, whose quantized scale
222225
# gradient is accumulated with atomics; TE raises when bit-exact execution is requested,
223226
# so these recipes must not pin NVTE_ALLOW_NONDETERMINISTIC_ALGO=0.

0 commit comments

Comments
 (0)