Skip to content

Commit 3dee1f2

Browse files
committed
fix(test): Rewrite tuner distributed integ test to match CI patterns
- Use sagemaker_session fixture from conftest (auto-resolves role/region) - Use ml.m5.xlarge CPU instance (cheaper, available in CI) - Remove hardcoded role ARN and training_mode - Remove @pytest.mark.slow (not registered in CI config) - Use module-level function instead of class (matches other integ tests) - Use DEFAULT_CPU_IMAGE consistent with test_model_trainer.py
1 parent 988a57e commit 3dee1f2

1 file changed

Lines changed: 74 additions & 85 deletions

File tree

‎sagemaker-train/tests/integ/train/test_tuner_distributed.py‎

Lines changed: 74 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -10,39 +10,36 @@
1010
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
1111
# ANY KIND, either express or implied. See the License for the specific
1212
# language governing permissions and limitations under the License.
13-
"""Integration test: HyperparameterTuner with Torchrun distributed training."""
13+
"""Integration test: HyperparameterTuner with Torchrun distributed training.
14+
15+
Regression test for the bug where HyperparameterTuner dropped the sm_drivers
16+
channel, causing the container to fall back to single-GPU execution instead
17+
of using torchrun for multi-GPU distributed training.
18+
"""
19+
from __future__ import absolute_import
1420

1521
import os
1622
import time
1723
import logging
24+
1825
import pytest
1926

20-
from sagemaker.train import ModelTrainer
21-
from sagemaker.train.model_trainer import Mode
27+
from sagemaker.train.model_trainer import ModelTrainer
2228
from sagemaker.train.configs import SourceCode, Compute
2329
from sagemaker.train.distributed import Torchrun
2430
from sagemaker.train.tuner import HyperparameterTuner
2531
from sagemaker.core.parameter import ContinuousParameter
26-
from sagemaker.core.shapes import StoppingCondition
2732

2833
logger = logging.getLogger(__name__)
2934

30-
TRAINING_IMAGE = (
31-
"763104351884.dkr.ecr.us-west-2.amazonaws.com/"
32-
"huggingface-pytorch-training:2.8.0-transformers4.56.2"
33-
"-gpu-py312-cu129-ubuntu22.04-v1.0"
35+
DATA_DIR = os.path.join(os.path.dirname(__file__), "../..", "data")
36+
DEFAULT_CPU_IMAGE = (
37+
"763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.0.0-cpu-py310"
3438
)
35-
ROLE = os.environ.get(
36-
"SAGEMAKER_ROLE_ARN",
37-
"arn:aws:iam::123456789012:role/SageMakerRole",
38-
)
39-
INSTANCE_TYPE = "ml.g5.12xlarge"
40-
4139

42-
TRAIN_SCRIPT = '''\
40+
TRAIN_SCRIPT_CONTENT = """\
4341
import os
4442
import argparse
45-
import torch
4643
4744
4845
def main():
@@ -53,86 +50,78 @@ def main():
5350
world_size = int(os.environ.get("WORLD_SIZE", 1))
5451
local_rank = int(os.environ.get("LOCAL_RANK", 0))
5552
56-
print(f"DISTRIBUTED_CHECK: world_size={world_size}, local_rank={local_rank}")
57-
print(f"DISTRIBUTED_CHECK: gpu_count={torch.cuda.device_count()}")
58-
59-
if world_size > 1:
60-
print("DISTRIBUTED_CHECK: MULTI_GPU_MODE=true")
61-
torch.distributed.init_process_group(backend="nccl")
62-
torch.distributed.destroy_process_group()
63-
else:
64-
print("DISTRIBUTED_CHECK: MULTI_GPU_MODE=false")
53+
print(f"DISTRIBUTED_CHECK: world_size={world_size}")
54+
print(f"DISTRIBUTED_CHECK: local_rank={local_rank}")
55+
print(f"DISTRIBUTED_CHECK: learning_rate={args.learning_rate}")
6556
6657
# Emit metric for the tuner to parse
67-
print("eval_loss: 0.5")
58+
print(f"eval_loss: 0.42")
6859
6960
7061
if __name__ == "__main__":
7162
main()
72-
'''
63+
"""
7364

7465

7566
@pytest.fixture(scope="module")
76-
def training_script_dir(tmp_path_factory):
67+
def train_source_dir(tmp_path_factory):
7768
"""Create a temp directory with a minimal training script."""
78-
d = tmp_path_factory.mktemp("src")
79-
(d / "train.py").write_text(TRAIN_SCRIPT)
69+
d = tmp_path_factory.mktemp("tuner_dist_src")
70+
(d / "train.py").write_text(TRAIN_SCRIPT_CONTENT)
8071
return str(d)
8172

8273

83-
@pytest.mark.slow
84-
class TestTunerDistributedTraining:
85-
"""Regression test: tuner must preserve Torchrun distributed config."""
86-
87-
def test_tuner_launches_torchrun(self, training_script_dir):
88-
"""Tuning job should use torchrun (multi-GPU), not single-GPU fallback."""
89-
model_trainer = ModelTrainer(
90-
training_mode=Mode.SAGEMAKER_TRAINING_JOB,
91-
role=ROLE,
92-
training_image=TRAINING_IMAGE,
93-
base_job_name=f"tuner-dist-{int(time.time())}",
94-
source_code=SourceCode(
95-
source_dir=training_script_dir,
96-
entry_script="train.py",
97-
),
98-
compute=Compute(
99-
instance_type=INSTANCE_TYPE,
100-
instance_count=1,
101-
volume_size_in_gb=30,
102-
),
103-
distributed=Torchrun(),
104-
stopping_condition=StoppingCondition(
105-
max_runtime_in_seconds=600,
74+
def test_tuner_includes_sm_drivers_channel(sagemaker_session, train_source_dir):
75+
"""Verify tuning jobs include sm_drivers channel for distributed training.
76+
77+
Uses a CPU instance with Torchrun to validate that the sm_drivers channel
78+
(containing torchrun_driver.py and sm_train.sh) is included in the tuning
79+
job definition. The training script logs WORLD_SIZE to confirm the V3
80+
driver path is used instead of the legacy framework container fallback.
81+
"""
82+
model_trainer = ModelTrainer(
83+
sagemaker_session=sagemaker_session,
84+
training_image=DEFAULT_CPU_IMAGE,
85+
base_job_name="tuner-dist-test",
86+
source_code=SourceCode(
87+
source_dir=train_source_dir,
88+
entry_script="train.py",
89+
),
90+
compute=Compute(
91+
instance_type="ml.m5.xlarge",
92+
instance_count=1,
93+
volume_size_in_gb=30,
94+
),
95+
distributed=Torchrun(),
96+
hyperparameters={"learning_rate": 1e-4},
97+
)
98+
99+
tuner = HyperparameterTuner(
100+
model_trainer=model_trainer,
101+
objective_metric_name="eval_loss",
102+
metric_definitions=[
103+
{"Name": "eval_loss", "Regex": r"eval_loss: ([0-9\\.]+)"},
104+
],
105+
hyperparameter_ranges={
106+
"learning_rate": ContinuousParameter(
107+
min_value=1e-5,
108+
max_value=5e-4,
109+
scaling_type="Logarithmic",
106110
),
107-
hyperparameters={"learning_rate": 1e-4},
108-
)
109-
110-
tuner = HyperparameterTuner(
111-
model_trainer=model_trainer,
112-
objective_metric_name="eval_loss",
113-
metric_definitions=[
114-
{"Name": "eval_loss", "Regex": r"eval_loss: ([0-9\\.]+)"},
115-
],
116-
hyperparameter_ranges={
117-
"learning_rate": ContinuousParameter(
118-
min_value=1e-5,
119-
max_value=5e-4,
120-
scaling_type="Logarithmic",
121-
),
122-
},
123-
objective_type="Minimize",
124-
max_jobs=1,
125-
max_parallel_jobs=1,
126-
)
127-
128-
tuner.tune(wait=True)
129-
130-
job = tuner.latest_tuning_job.refresh()
131-
assert job.hyper_parameter_tuning_job_status in (
132-
"Completed",
133-
"Stopped",
134-
), f"Tuning job failed: {job.hyper_parameter_tuning_job_status}"
135-
136-
best = tuner.best_training_job()
137-
assert best is not None
138-
logger.info("PASSED: tuner distributed training test - job: %s", best)
111+
},
112+
objective_type="Minimize",
113+
max_jobs=1,
114+
max_parallel_jobs=1,
115+
)
116+
117+
tuner.tune(wait=True)
118+
119+
job = tuner.latest_tuning_job.refresh()
120+
assert job.hyper_parameter_tuning_job_status in (
121+
"Completed",
122+
"Stopped",
123+
), f"Tuning job failed: {job.hyper_parameter_tuning_job_status}"
124+
125+
best = tuner.best_training_job()
126+
assert best is not None
127+
logger.info("PASSED: tuner distributed training test - job: %s", best)

0 commit comments

Comments
 (0)