Skip to content

Commit c9886ea

Browse files
authored
test(train): Bound capacity wait in SMTJ SFT integ test (#6409)
test_sft_trainer_serverful_smtj trains on one ml.g6.12xlarge in us-east-1. When there is no capacity the job stays in Pending, and the test polled for up to 3 hours, which held the GPU integ build until CodeBuild timed out. The job was then left queued behind the killed build and trained hours later with nobody waiting for it. Poll with the shared bounded capacity wait from tests/integ/capacity.py: after 90 minutes in Pending the job is stopped and the test is skipped, and the job is stopped whenever the wait ends with it still running. The total wait is 2 hours, covering the capacity wait plus the job's ~21 minute run time, so the test stays well inside the build timeout.
1 parent 677d42d commit c9886ea

1 file changed

Lines changed: 25 additions & 21 deletions

File tree

‎sagemaker-train/tests/integ/train/test_sft_trainer_serverful_smtj.py‎

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,20 @@
4141
from sagemaker.train.common import TrainingType
4242
from sagemaker.core.training.configs import TrainingJobCompute
4343

44+
from ..capacity import wait_for_training_job_with_capacity_timeout
45+
4446
logger = logging.getLogger(__name__)
4547

4648
# Test configuration
49+
SMTJ_INSTANCE_TYPE = "ml.g6.12xlarge"
50+
# In us-east-1, SMTJ jobs for this test waited 0 min to 48 h for an ml.g6.12xlarge
51+
# (median ~2 h); once the instance is acquired, download + training + upload take
52+
# <= ~21 min. A job still waiting after 90 min is treated as a capacity shortage.
53+
# The total bound covers that wait plus the job's own run time, and keeps the test
54+
# well inside the CI build timeout.
55+
SMTJ_PENDING_TIMEOUT_SECONDS = 90 * 60
56+
SMTJ_MAX_WAIT_SECONDS = 2 * 3600
57+
4758
DATA_PREFIX = "sft-smtj-integ"
4859
DATA_S3_KEY = f"{DATA_PREFIX}/sft_smtj_sample_data.jsonl"
4960

@@ -116,7 +127,7 @@ def test_sft_trainer_serverful_smtj(sagemaker_session_us_east_1, training_resour
116127
training_dataset=training_resources["training_dataset"],
117128
s3_output_path=training_resources["s3_output_path"],
118129
compute=TrainingJobCompute(
119-
instance_type="ml.g6.12xlarge",
130+
instance_type=SMTJ_INSTANCE_TYPE,
120131
instance_count=1,
121132
),
122133
sagemaker_session=sagemaker_session_us_east_1,
@@ -140,27 +151,20 @@ def test_sft_trainer_serverful_smtj(sagemaker_session_us_east_1, training_resour
140151
assert training_job.training_job_name is not None
141152
logger.info(f"Training job submitted: {training_job.training_job_name}")
142153

143-
# Poll for completion
144-
max_wait_time = 10800 # 3 hour timeout (Nova training can be slow)
145-
poll_interval = 60
146-
start_time = time.time()
147-
148-
while time.time() - start_time < max_wait_time:
149-
training_job.refresh()
150-
status = training_job.training_job_status
151-
152-
if status in ["Completed", "Failed", "Stopped"]:
153-
break
154-
155-
logger.info(
156-
f"Job {training_job.training_job_name} status: {status} "
157-
f"({int(time.time() - start_time)}s elapsed)"
158-
)
159-
time.sleep(poll_interval)
154+
# Poll for completion with a bounded wait for capacity. On a capacity shortage
155+
# the job is stopped and the test is skipped, so it neither holds the build
156+
# until CodeBuild times out nor leaves the job queued behind a killed build.
157+
status = wait_for_training_job_with_capacity_timeout(
158+
training_job,
159+
max_wait_time=SMTJ_MAX_WAIT_SECONDS,
160+
poll_interval=60,
161+
pending_timeout=SMTJ_PENDING_TIMEOUT_SECONDS,
162+
capacity_label=SMTJ_INSTANCE_TYPE,
163+
)
160164

161-
assert training_job.training_job_status == "Completed", (
162-
f"Training job {training_job.training_job_name} ended with status: "
163-
f"{training_job.training_job_status}"
165+
assert status == "Completed", (
166+
f"Training job {training_job.training_job_name} ended with status {status}: "
167+
f"{training_job.failure_reason}"
164168
)
165169
logger.info(f"Training job completed successfully: {training_job.training_job_name}")
166170

0 commit comments

Comments
 (0)