Skip to content

Commit 4f4f488

Browse files
authored
test(train): Bound capacity wait in gated JumpStart train test (#6397)
test_jumpstart_train_from_gated_reference trains on the gated model's default GPU instance (ml.g5.2xlarge) with train(wait=True). When there is no capacity the job stays in Pending ("waiting for capacity") indefinitely: MaxRuntimeInSeconds does not count queue time and TrainingJob.wait() has no timeout. The test then blocked until the CI Health V3 release canary build timed out, and the queued job was left behind holding the account's single ml.g5.2xlarge training quota, which in turn failed the V2 slow test suite with ResourceLimitExceeded. Submit with wait=False and poll with a bounded capacity wait: after an hour in Pending the job is stopped and the test is skipped. The total wait covers queue time plus the job's MaxRuntimeInSeconds, so the test never ends before the job does. Move the bounded wait out of the Nova-only helper into a shared tests/integ/capacity.py. wait_for_nova_training_job keeps its signature and defaults and now delegates to the shared helper.
1 parent 4ff8e11 commit 4f4f488

3 files changed

Lines changed: 149 additions & 63 deletions

File tree

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Helpers for integ tests that submit real training jobs on scarce instances.
14+
15+
When SageMaker has no capacity for the requested instance type, a training job
16+
does not fail: it stays in secondary status ``Pending`` ("Training job waiting
17+
for capacity") indefinitely. ``MaxRuntimeInSeconds`` only counts training time,
18+
so it does not bound that wait, and ``TrainingJob.wait()`` has no timeout. A
19+
test that calls ``train()`` with the default ``wait=True`` therefore blocks
20+
until the CodeBuild build times out, and the queued job is left behind holding
21+
the account's instance quota after the build is gone.
22+
23+
``wait_for_training_job_with_capacity_timeout`` waits with a bounded wait for
24+
capacity instead: if the job is still waiting for instances after
25+
``pending_timeout`` it is stopped and the test is skipped. It also stops the job
26+
whenever the wait ends with the job still running (timeout, error, interrupt),
27+
so no orphaned job keeps queueing for capacity after the test is gone.
28+
"""
29+
30+
from __future__ import absolute_import
31+
32+
import logging
33+
import time
34+
35+
import pytest
36+
37+
logger = logging.getLogger(__name__)
38+
39+
TERMINAL_STATUSES = ("Completed", "Failed", "Stopped")
40+
_WAITING_FOR_CAPACITY_STATUSES = ("Starting", "Pending")
41+
42+
DEFAULT_PENDING_TIMEOUT_SECONDS = 3600
43+
44+
45+
def stop_training_job_quietly(training_job):
46+
"""Best-effort stop of a training job that is not terminal yet."""
47+
name = getattr(training_job, "training_job_name", None)
48+
try:
49+
logger.warning("Stopping non-terminal training job %s", name)
50+
training_job.stop()
51+
except Exception as e: # pylint: disable=broad-except
52+
logger.warning("Failed to stop training job %s: %s", name, e)
53+
54+
55+
def wait_for_training_job_with_capacity_timeout(
56+
training_job,
57+
max_wait_time,
58+
poll_interval=30,
59+
pending_timeout=DEFAULT_PENDING_TIMEOUT_SECONDS,
60+
capacity_label="Training",
61+
):
62+
"""Poll a training job until it is terminal, with a bounded wait for capacity.
63+
64+
Args:
65+
training_job: The ``sagemaker.core.resources.TrainingJob`` that was
66+
submitted without waiting (``train(wait=False)``).
67+
max_wait_time (int): Maximum total seconds to wait for a terminal status.
68+
poll_interval (int): Seconds between status checks.
69+
pending_timeout (int): Seconds after which a job that is still waiting for
70+
instances (secondary status ``Starting``/``Pending``) is stopped and the
71+
test is skipped.
72+
capacity_label (str): Prefix for the skip message, naming the capacity
73+
that was unavailable (e.g. ``"Nova"`` or ``"ml.g5.2xlarge"``).
74+
75+
Returns:
76+
str: The last observed ``training_job_status``. It is terminal unless
77+
``max_wait_time`` ran out, in which case the job has been stopped.
78+
"""
79+
start = time.time()
80+
status = None
81+
try:
82+
while time.time() - start < max_wait_time:
83+
training_job.refresh()
84+
status = training_job.training_job_status
85+
if status in TERMINAL_STATUSES:
86+
return status
87+
elapsed = time.time() - start
88+
if (
89+
training_job.secondary_status in _WAITING_FOR_CAPACITY_STATUSES
90+
and elapsed > pending_timeout
91+
):
92+
pytest.skip(
93+
f"{capacity_label} capacity unavailable: training job "
94+
f"{training_job.training_job_name} was still waiting for instances "
95+
f"after {int(elapsed)}s; stopped it to release the capacity."
96+
)
97+
time.sleep(poll_interval)
98+
logger.warning(
99+
"Training job %s not terminal after %ds (status: %s)",
100+
training_job.training_job_name,
101+
max_wait_time,
102+
status,
103+
)
104+
return status
105+
finally:
106+
if status not in TERMINAL_STATUSES:
107+
stop_training_job_quietly(training_job)

‎sagemaker-train/tests/integ/jumpstart/test_jumpstart_train.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
from sagemaker.train import ModelTrainer
3838
from sagemaker.train.configs import Compute
3939

40+
from ..capacity import wait_for_training_job_with_capacity_timeout
41+
4042
logger = logging.getLogger(__name__)
4143

4244
# A trainable classical-ML model keeps these tests fast/cheap on CPU.
@@ -50,6 +52,13 @@
5052
# SupportedTrainingInstanceTypes and raises if it is not in the list, so
5153
# resolving the model's own default is safer than hardcoding one here.
5254
GATED_TRAINABLE_MODEL_ID = "meta-textgeneration-llama-3-2-1b"
55+
# In CI the gated job has acquired its default GPU (ml.g5.2xlarge) after ~17 min
56+
# when capacity exists; a job still waiting after an hour is treated as a
57+
# capacity shortage. The total bound covers that wait plus the job's own
58+
# MaxRuntimeInSeconds (3600) and instance setup, so the test can never outlive
59+
# the job and leave it queued behind a killed build.
60+
GATED_TRAINING_PENDING_TIMEOUT_SECONDS = 3600
61+
GATED_TRAINING_MAX_WAIT_SECONDS = 2 * 3600 + 900
5362
HUB_NAME_PREFIX = "sdk-integ-train-hub"
5463
ALIASED_REFERENCE_NAME = "sdk-integ-aliased-catboost"
5564
PRIVATE_MODEL_NAME = "sdk-integ-private-catboost"
@@ -361,7 +370,24 @@ def test_jumpstart_train_from_gated_reference(private_hub, sagemaker_session):
361370
# Train on the SDK-resolved channels (no explicit training channel), so the
362371
# hub-aware, gated channel construction under test is actually exercised
363372
# end-to-end against a real training job.
364-
model_trainer.train()
373+
#
374+
# The gated model's default GPU instance can sit in "waiting for capacity"
375+
# indefinitely, and train(wait=True) has no bound on that wait, so submit
376+
# without waiting and poll with a bounded capacity wait. On a capacity
377+
# shortage the job is stopped (releasing the instance quota) and the test is
378+
# skipped; the channel assertions above have already run.
379+
model_trainer.train(wait=False)
380+
training_job = model_trainer._latest_training_job
381+
status = wait_for_training_job_with_capacity_timeout(
382+
training_job,
383+
max_wait_time=GATED_TRAINING_MAX_WAIT_SECONDS,
384+
pending_timeout=GATED_TRAINING_PENDING_TIMEOUT_SECONDS,
385+
capacity_label=model_trainer.compute.instance_type,
386+
)
387+
assert status == "Completed", (
388+
f"Training job {training_job.training_job_name} ended with status {status}: "
389+
f"{training_job.failure_reason}"
390+
)
365391

366392

367393
def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):

‎sagemaker-train/tests/integ/train/nova_capacity.py‎

Lines changed: 15 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,10 @@
2020
2121
* ``nova_capacity_slot`` serializes Nova serverless jobs across all xdist workers of a
2222
test run. Hold it from submission until the job is terminal.
23-
* ``wait_for_nova_training_job`` waits for the job with a bounded wait for capacity:
24-
if the job is still waiting for instances after ``pending_timeout`` it is stopped
25-
and the test is skipped. It also stops the job whenever the wait ends with the job
26-
still running (timeout, error, interrupt), so no orphaned job keeps queueing for
27-
capacity after the test is gone.
23+
* ``wait_for_nova_training_job`` waits for the job with a bounded wait for capacity
24+
(see ``tests/integ/capacity.py``): if the job is still waiting for instances after
25+
``pending_timeout`` it is stopped and the test is skipped, and the job is stopped
26+
whenever the wait ends with it still running.
2827
"""
2928

3029
from __future__ import absolute_import
@@ -35,9 +34,8 @@
3534
import time
3635
from contextlib import contextmanager
3736

38-
import pytest
39-
4037
from .. import lock
38+
from ..capacity import wait_for_training_job_with_capacity_timeout
4139

4240
logger = logging.getLogger(__name__)
4341

@@ -51,9 +49,6 @@
5149
# give the capacity back instead of queueing behind customers.
5250
NOVA_PENDING_TIMEOUT_SECONDS = 3600
5351

54-
TERMINAL_STATUSES = ("Completed", "Failed", "Stopped")
55-
_WAITING_FOR_CAPACITY_STATUSES = ("Starting", "Pending")
56-
5752

5853
@contextmanager
5954
def nova_capacity_slot():
@@ -65,63 +60,21 @@ def nova_capacity_slot():
6560
yield
6661

6762

68-
def _stop_quietly(training_job):
69-
"""Best-effort stop of a training job that is not terminal yet."""
70-
name = getattr(training_job, "training_job_name", None)
71-
try:
72-
logger.warning("Stopping non-terminal training job %s", name)
73-
training_job.stop()
74-
except Exception as e: # pylint: disable=broad-except
75-
logger.warning("Failed to stop training job %s: %s", name, e)
76-
77-
7863
def wait_for_nova_training_job(
7964
training_job,
8065
max_wait_time,
8166
poll_interval=30,
8267
pending_timeout=NOVA_PENDING_TIMEOUT_SECONDS,
8368
):
84-
"""Poll a training job until it is terminal, with a bounded wait for capacity.
85-
86-
Args:
87-
training_job: The ``sagemaker.core.resources.TrainingJob`` returned by
88-
``train(wait=False)``.
89-
max_wait_time (int): Maximum total seconds to wait for a terminal status.
90-
poll_interval (int): Seconds between status checks.
91-
pending_timeout (int): Seconds after which a job that is still waiting for
92-
instances (secondary status ``Starting``/``Pending``) is stopped and the
93-
test is skipped.
69+
"""Poll a Nova training job until it is terminal, with a bounded wait for capacity.
9470
95-
Returns:
96-
str: The last observed ``training_job_status``. It is terminal unless
97-
``max_wait_time`` ran out, in which case the job has been stopped.
71+
Thin wrapper over ``wait_for_training_job_with_capacity_timeout``; see that
72+
function for the arguments and return value.
9873
"""
99-
start = time.time()
100-
status = None
101-
try:
102-
while time.time() - start < max_wait_time:
103-
training_job.refresh()
104-
status = training_job.training_job_status
105-
if status in TERMINAL_STATUSES:
106-
return status
107-
elapsed = time.time() - start
108-
if (
109-
training_job.secondary_status in _WAITING_FOR_CAPACITY_STATUSES
110-
and elapsed > pending_timeout
111-
):
112-
pytest.skip(
113-
f"Nova capacity unavailable: training job "
114-
f"{training_job.training_job_name} was still waiting for instances "
115-
f"after {int(elapsed)}s; stopped it to release the shared pool."
116-
)
117-
time.sleep(poll_interval)
118-
logger.warning(
119-
"Training job %s not terminal after %ds (status: %s)",
120-
training_job.training_job_name,
121-
max_wait_time,
122-
status,
123-
)
124-
return status
125-
finally:
126-
if status not in TERMINAL_STATUSES:
127-
_stop_quietly(training_job)
74+
return wait_for_training_job_with_capacity_timeout(
75+
training_job,
76+
max_wait_time=max_wait_time,
77+
poll_interval=poll_interval,
78+
pending_timeout=pending_timeout,
79+
capacity_label="Nova",
80+
)

0 commit comments

Comments
 (0)