Skip to content

Commit d0abf32

Browse files
fix: Add ml.p5e.48xlarge and ml.p5.48xlarge to EFA instance lists (#5492)
Add ml.p5e.48xlarge to SM_EFA_NCCL_INSTANCES and SM_EFA_RDMA_INSTANCES. Add ml.p5.48xlarge to SM_EFA_RDMA_INSTANCES (was missing). Without these entries, NCCL hangs during distributed training initialization on P5e instances due to missing EFA environment variables (FI_PROVIDER, FI_EFA_USE_DEVICE_RDMA, RDMAV_FORK_SAFE). Fixes #5491
1 parent 9a84bff commit d0abf32

5 files changed

Lines changed: 38 additions & 0 deletions

File tree

‎sagemaker-core/src/sagemaker/core/modules/train/container_drivers/common/utils.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,15 @@
5151
"ml.p4d.24xlarge",
5252
"ml.p4de.24xlarge",
5353
"ml.p5.48xlarge",
54+
"ml.p5e.48xlarge",
5455
"ml.trn1.32xlarge",
5556
]
5657

5758
SM_EFA_RDMA_INSTANCES = [
5859
"ml.p4d.24xlarge",
5960
"ml.p4de.24xlarge",
61+
"ml.p5.48xlarge",
62+
"ml.p5e.48xlarge",
6063
"ml.trn1.32xlarge",
6164
]
6265

‎sagemaker-core/src/sagemaker/core/remote_function/runtime_environment/bootstrap_runtime_environment.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,15 @@
7676
"ml.p4d.24xlarge",
7777
"ml.p4de.24xlarge",
7878
"ml.p5.48xlarge",
79+
"ml.p5e.48xlarge",
7980
"ml.trn1.32xlarge",
8081
]
8182

8283
SM_EFA_RDMA_INSTANCES = [
8384
"ml.p4d.24xlarge",
8485
"ml.p4de.24xlarge",
86+
"ml.p5.48xlarge",
87+
"ml.p5e.48xlarge",
8588
"ml.trn1.32xlarge",
8689
]
8790

‎sagemaker-train/src/sagemaker/train/container_drivers/common/utils.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,15 @@
5151
"ml.p4d.24xlarge",
5252
"ml.p4de.24xlarge",
5353
"ml.p5.48xlarge",
54+
"ml.p5e.48xlarge",
5455
"ml.trn1.32xlarge",
5556
]
5657

5758
SM_EFA_RDMA_INSTANCES = [
5859
"ml.p4d.24xlarge",
5960
"ml.p4de.24xlarge",
61+
"ml.p5.48xlarge",
62+
"ml.p5e.48xlarge",
6063
"ml.trn1.32xlarge",
6164
]
6265

‎sagemaker-train/src/sagemaker/train/remote_function/runtime_environment/bootstrap_runtime_environment.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,15 @@
7676
"ml.p4d.24xlarge",
7777
"ml.p4de.24xlarge",
7878
"ml.p5.48xlarge",
79+
"ml.p5e.48xlarge",
7980
"ml.trn1.32xlarge",
8081
]
8182

8283
SM_EFA_RDMA_INSTANCES = [
8384
"ml.p4d.24xlarge",
8485
"ml.p4de.24xlarge",
86+
"ml.p5.48xlarge",
87+
"ml.p5e.48xlarge",
8588
"ml.trn1.32xlarge",
8689
]
8790

‎sagemaker-train/tests/unit/train/container_drivers/test_torchrun_driver.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import sys
1919
import json
2020

21+
import pytest
2122
from unittest.mock import patch, MagicMock
2223

2324
sys.modules["utils"] = MagicMock()
@@ -147,3 +148,28 @@ def test_create_commands_multi_node(
147148

148149
command = torchrun_driver.create_commands()
149150
assert command == expected_command
151+
152+
153+
@pytest.mark.parametrize("instance_type", ["ml.p5.48xlarge", "ml.p5e.48xlarge"])
154+
@patch.dict(
155+
os.environ,
156+
{
157+
"SM_NETWORK_INTERFACE_NAME": "eth0",
158+
"SM_HOST_COUNT": "2",
159+
"SM_MASTER_ADDR": "algo-1",
160+
"SM_MASTER_PORT": "7777",
161+
"SM_CURRENT_HOST_RANK": "0",
162+
"SM_HPS": json.dumps({}),
163+
"SM_DISTRIBUTED_CONFIG": json.dumps(DUMMY_DISTRIBUTED),
164+
"SM_ENTRY_SCRIPT": "script.py",
165+
},
166+
)
167+
def test_p5_p5e_efa_environment_setup(instance_type):
168+
"""Test that P5 and P5e instances are in EFA instance lists."""
169+
from sagemaker.train.container_drivers.common.utils import (
170+
SM_EFA_NCCL_INSTANCES,
171+
SM_EFA_RDMA_INSTANCES,
172+
)
173+
174+
assert instance_type in SM_EFA_NCCL_INSTANCES
175+
assert instance_type in SM_EFA_RDMA_INSTANCES

0 commit comments

Comments
 (0)