1010# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
1111# ANY KIND, either express or implied. See the License for the specific
1212# language governing permissions and limitations under the License.
13- """Integration test: HyperparameterTuner with Torchrun distributed training."""
13+ """Integration test: HyperparameterTuner with Torchrun distributed training.
14+
15+ Regression test for the bug where HyperparameterTuner dropped the sm_drivers
16+ channel, causing the container to fall back to single-GPU execution instead
17+ of using torchrun for multi-GPU distributed training.
18+ """
19+ from __future__ import absolute_import
1420
1521import os
1622import time
1723import logging
24+
1825import pytest
1926
20- from sagemaker .train import ModelTrainer
21- from sagemaker .train .model_trainer import Mode
27+ from sagemaker .train .model_trainer import ModelTrainer
2228from sagemaker .train .configs import SourceCode , Compute
2329from sagemaker .train .distributed import Torchrun
2430from sagemaker .train .tuner import HyperparameterTuner
2531from sagemaker .core .parameter import ContinuousParameter
26- from sagemaker .core .shapes import StoppingCondition
2732
2833logger = logging .getLogger (__name__ )
2934
30- TRAINING_IMAGE = (
31- "763104351884.dkr.ecr.us-west-2.amazonaws.com/"
32- "huggingface-pytorch-training:2.8.0-transformers4.56.2"
33- "-gpu-py312-cu129-ubuntu22.04-v1.0"
35+ DATA_DIR = os .path .join (os .path .dirname (__file__ ), "../.." , "data" )
36+ DEFAULT_CPU_IMAGE = (
37+ "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:2.0.0-cpu-py310"
3438)
35- ROLE = os .environ .get (
36- "SAGEMAKER_ROLE_ARN" ,
37- "arn:aws:iam::123456789012:role/SageMakerRole" ,
38- )
39- INSTANCE_TYPE = "ml.g5.12xlarge"
40-
4139
42- TRAIN_SCRIPT = ''' \
40+ TRAIN_SCRIPT_CONTENT = """ \
4341 import os
4442import argparse
45- import torch
4643
4744
4845def main():
@@ -53,86 +50,78 @@ def main():
5350 world_size = int(os.environ.get("WORLD_SIZE", 1))
5451 local_rank = int(os.environ.get("LOCAL_RANK", 0))
5552
56- print(f"DISTRIBUTED_CHECK: world_size={world_size}, local_rank={local_rank}")
57- print(f"DISTRIBUTED_CHECK: gpu_count={torch.cuda.device_count()}")
58-
59- if world_size > 1:
60- print("DISTRIBUTED_CHECK: MULTI_GPU_MODE=true")
61- torch.distributed.init_process_group(backend="nccl")
62- torch.distributed.destroy_process_group()
63- else:
64- print("DISTRIBUTED_CHECK: MULTI_GPU_MODE=false")
53+ print(f"DISTRIBUTED_CHECK: world_size={world_size}")
54+ print(f"DISTRIBUTED_CHECK: local_rank={local_rank}")
55+ print(f"DISTRIBUTED_CHECK: learning_rate={args.learning_rate}")
6556
6657 # Emit metric for the tuner to parse
67- print("eval_loss: 0.5 ")
58+ print(f "eval_loss: 0.42 ")
6859
6960
7061if __name__ == "__main__":
7162 main()
72- '''
63+ """
7364
7465
7566@pytest .fixture (scope = "module" )
76- def training_script_dir (tmp_path_factory ):
67+ def train_source_dir (tmp_path_factory ):
7768 """Create a temp directory with a minimal training script."""
78- d = tmp_path_factory .mktemp ("src " )
79- (d / "train.py" ).write_text (TRAIN_SCRIPT )
69+ d = tmp_path_factory .mktemp ("tuner_dist_src " )
70+ (d / "train.py" ).write_text (TRAIN_SCRIPT_CONTENT )
8071 return str (d )
8172
8273
83- @pytest .mark .slow
84- class TestTunerDistributedTraining :
85- """Regression test: tuner must preserve Torchrun distributed config."""
86-
87- def test_tuner_launches_torchrun (self , training_script_dir ):
88- """Tuning job should use torchrun (multi-GPU), not single-GPU fallback."""
89- model_trainer = ModelTrainer (
90- training_mode = Mode .SAGEMAKER_TRAINING_JOB ,
91- role = ROLE ,
92- training_image = TRAINING_IMAGE ,
93- base_job_name = f"tuner-dist-{ int (time .time ())} " ,
94- source_code = SourceCode (
95- source_dir = training_script_dir ,
96- entry_script = "train.py" ,
97- ),
98- compute = Compute (
99- instance_type = INSTANCE_TYPE ,
100- instance_count = 1 ,
101- volume_size_in_gb = 30 ,
102- ),
103- distributed = Torchrun (),
104- stopping_condition = StoppingCondition (
105- max_runtime_in_seconds = 600 ,
74+ def test_tuner_includes_sm_drivers_channel (sagemaker_session , train_source_dir ):
75+ """Verify tuning jobs include sm_drivers channel for distributed training.
76+
77+ Uses a CPU instance with Torchrun to validate that the sm_drivers channel
78+ (containing torchrun_driver.py and sm_train.sh) is included in the tuning
79+ job definition. The training script logs WORLD_SIZE to confirm the V3
80+ driver path is used instead of the legacy framework container fallback.
81+ """
82+ model_trainer = ModelTrainer (
83+ sagemaker_session = sagemaker_session ,
84+ training_image = DEFAULT_CPU_IMAGE ,
85+ base_job_name = "tuner-dist-test" ,
86+ source_code = SourceCode (
87+ source_dir = train_source_dir ,
88+ entry_script = "train.py" ,
89+ ),
90+ compute = Compute (
91+ instance_type = "ml.m5.xlarge" ,
92+ instance_count = 1 ,
93+ volume_size_in_gb = 30 ,
94+ ),
95+ distributed = Torchrun (),
96+ hyperparameters = {"learning_rate" : 1e-4 },
97+ )
98+
99+ tuner = HyperparameterTuner (
100+ model_trainer = model_trainer ,
101+ objective_metric_name = "eval_loss" ,
102+ metric_definitions = [
103+ {"Name" : "eval_loss" , "Regex" : r"eval_loss: ([0-9\\.]+)" },
104+ ],
105+ hyperparameter_ranges = {
106+ "learning_rate" : ContinuousParameter (
107+ min_value = 1e-5 ,
108+ max_value = 5e-4 ,
109+ scaling_type = "Logarithmic" ,
106110 ),
107- hyperparameters = {"learning_rate" : 1e-4 },
108- )
109-
110- tuner = HyperparameterTuner (
111- model_trainer = model_trainer ,
112- objective_metric_name = "eval_loss" ,
113- metric_definitions = [
114- {"Name" : "eval_loss" , "Regex" : r"eval_loss: ([0-9\\.]+)" },
115- ],
116- hyperparameter_ranges = {
117- "learning_rate" : ContinuousParameter (
118- min_value = 1e-5 ,
119- max_value = 5e-4 ,
120- scaling_type = "Logarithmic" ,
121- ),
122- },
123- objective_type = "Minimize" ,
124- max_jobs = 1 ,
125- max_parallel_jobs = 1 ,
126- )
127-
128- tuner .tune (wait = True )
129-
130- job = tuner .latest_tuning_job .refresh ()
131- assert job .hyper_parameter_tuning_job_status in (
132- "Completed" ,
133- "Stopped" ,
134- ), f"Tuning job failed: { job .hyper_parameter_tuning_job_status } "
135-
136- best = tuner .best_training_job ()
137- assert best is not None
138- logger .info ("PASSED: tuner distributed training test - job: %s" , best )
111+ },
112+ objective_type = "Minimize" ,
113+ max_jobs = 1 ,
114+ max_parallel_jobs = 1 ,
115+ )
116+
117+ tuner .tune (wait = True )
118+
119+ job = tuner .latest_tuning_job .refresh ()
120+ assert job .hyper_parameter_tuning_job_status in (
121+ "Completed" ,
122+ "Stopped" ,
123+ ), f"Tuning job failed: { job .hyper_parameter_tuning_job_status } "
124+
125+ best = tuner .best_training_job ()
126+ assert best is not None
127+ logger .info ("PASSED: tuner distributed training test - job: %s" , best )
0 commit comments