Skip to content

Commit 32368c6

Browse files
bhargaveedeclaude
andcommitted
fix(perf): unset NCCL_NET_PLUGIN and tune B200 DSv3 256GPU recipe for NCCLEP rc5
- Remove NCCL_NET_PLUGIN from env to avoid plugin interference with NCCL EP - Set moe_use_grouped_tensor=False for 256GPU B200 fp8mx config - Add graph_capture_record_stream_reuse:True to PYTORCH_CUDA_ALLOC_CONF - Remove NCCL_EP_HT_EM_PULL_PUSH env var Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent fb0f78c commit 32368c6

2 files changed

Lines changed: 3 additions & 3 deletions

File tree

‎scripts/performance/bootstrap.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ def _apply_recipe_environment(recipe) -> None:
5252
if not isinstance(value, (str, int, float, bool)):
5353
raise TypeError(f"Environment variable {name!r} must have a scalar value, got {type(value).__name__}.")
5454
os.environ.setdefault(name, str(value))
55+
os.environ.pop("NCCL_NET_PLUGIN", None)
5556

5657

5758
def uses_ncclep(recipe) -> bool:

‎src/megatron/bridge/perf_recipes/deepseek/b200/deepseek_v3.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -152,20 +152,19 @@ def deepseek_v3_pretrain_256gpu_b200_fp8mx_config() -> ConfigContainer:
152152
cfg.comm_overlap.overlap_moe_expert_parallel_comm = True
153153
# Device-side expert token counts: the legacy grouped MLP path syncs tokens_per_expert to the
154154
# host every layer, which serializes the CPU behind the GPU when dispatch is fast.
155-
cfg.model.moe_use_grouped_tensor = True
155+
cfg.model.moe_use_grouped_tensor = False
156156
# Keep process settings next to the recipe so users can see the exact benchmark environment.
157157
cfg.env_vars = {
158158
**COMMON_PERF_ENV_VARS,
159159
# CUDA stream scheduling for this model and parallel layout.
160160
"CUDA_DEVICE_MAX_CONNECTIONS": 32,
161161
# CUDA graph and allocator behavior for this recipe.
162162
"NCCL_GRAPH_REGISTER": 0,
163-
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
163+
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True,graph_capture_record_stream_reuse:True",
164164
"TORCH_NCCL_AVOID_RECORD_STREAMS": 1,
165165
# NCCL user-buffer and launch settings.
166166
"NCCL_NVLS_ENABLE": 0,
167167
# NCCL EP dispatcher mode and one GPU per rank.
168-
"NCCL_EP_HT_EM_PULL_PUSH": 1,
169168
# Transformer Engine overlap settings for this model.
170169
"NVTE_BWD_LAYERNORM_SM_MARGIN": 20,
171170
"NVTE_FWD_LAYERNORM_SM_MARGIN": 20,

0 commit comments

Comments
 (0)