Skip to content

Commit cb24680

Browse files
committed
fix: treat Updating as in progress in core live-logging wait
Session.wait_for_endpoint(live_logging=True) stopped waiting as soon as the endpoint left Creating, so an endpoint that was Updating was reported as a failed deployment. Use the same in-progress statuses as _deploy_done (Creating, Updating), which the sagemaker-serve live-logging check already does. Also state that the ResourceIterator repeated-token guard only catches a NextToken that comes back unchanged, the same check botocore paginators do.
1 parent 02343e4 commit cb24680

3 files changed

Lines changed: 20 additions & 12 deletions

File tree

‎sagemaker-core/src/sagemaker/core/helper/session_helper.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3013,10 +3013,11 @@ def reset(self):
30133013
def _live_logging_deploy_done(
30143014
sagemaker_client, endpoint_name, paginator, paginator_config, poll, not_found_budget=None
30153015
):
3016-
"""Return the ``DescribeEndpoint`` response once the endpoint leaves ``Creating``.
3016+
"""Return the ``DescribeEndpoint`` response once the endpoint is no longer in progress.
30173017
30183018
Streams the endpoint's CloudWatch logs on every poll and returns ``None`` while
3019-
the endpoint is still being created. The log group only exists once a container
3019+
the endpoint is still ``Creating`` or ``Updating``, the same in-progress statuses
3020+
as ``_deploy_done``. The log group only exists once a container
30203021
has started, so an endpoint that fails before any instance is provisioned (for
30213022
example on InsufficientInstanceCapacity) never gets one: a missing log group
30223023
must not keep a finished deployment waiting.
@@ -3026,6 +3027,7 @@ def _live_logging_deploy_done(
30263027
consecutive polls a missing endpoint is tolerated. Without it a
30273028
missing endpoint is waited on indefinitely.
30283029
"""
3030+
in_progress_statuses = ["Creating", "Updating"]
30293031
stop = False
30303032
endpoint_status = None
30313033
try:
@@ -3042,7 +3044,7 @@ def _live_logging_deploy_done(
30423044
not_found_budget.reset()
30433045

30443046
# if endpoint is in an invalid state -> set stop to true, sleep, and flush the logs
3045-
if endpoint_status != "Creating":
3047+
if endpoint_status not in in_progress_statuses:
30463048
stop = True
30473049
if endpoint_status == "InService":
30483050
LOGGER.info(

‎sagemaker-core/src/sagemaker/core/utils/utils.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,8 @@ def __next__(self) -> T:
524524
# A page can be empty and still carry a NextToken (the service
525525
# filters results after paginating), so an empty page only ends
526526
# the iteration when there is no further page. A NextToken that
527-
# repeats would page forever, so it also ends the iteration.
527+
# comes back unchanged would page forever, so it also ends the
528+
# iteration (the same consecutive-repeat check botocore paginators do).
528529
if not self.next_token or self.next_token == previous_token:
529530
raise StopIteration
530531

‎sagemaker-core/tests/unit/helper/test_session_helper.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -429,14 +429,19 @@ def test_finished_endpoint_without_log_group_returns_desc(self, mock_sleep):
429429

430430
assert _live_logging_deploy_done(client, "my-endpoint", paginator, {}, 5) == desc
431431

432-
def test_creating_endpoint_without_log_group_keeps_waiting(self):
433-
"""Creating with no log group yet is still in progress."""
434-
client = Mock()
435-
client.describe_endpoint.return_value = {"EndpointStatus": "Creating"}
436-
paginator = Mock()
437-
paginator.paginate.side_effect = self.RESOURCE_NOT_FOUND
438-
439-
assert _live_logging_deploy_done(client, "my-endpoint", paginator, {}, 5) is None
432+
@patch("time.sleep")
433+
def test_in_progress_endpoint_keeps_waiting(self, mock_sleep):
434+
"""Creating or Updating is still in progress, with or without a log group."""
435+
for status in ("Creating", "Updating"):
436+
for log_error in (self.RESOURCE_NOT_FOUND, None):
437+
client = Mock()
438+
client.describe_endpoint.return_value = {"EndpointStatus": status}
439+
paginator = Mock()
440+
paginator.paginate.side_effect = log_error
441+
paginator.paginate.return_value = []
442+
443+
assert _live_logging_deploy_done(client, "my-endpoint", paginator, {}, 5) is None
444+
mock_sleep.assert_not_called()
440445

441446
def test_other_log_errors_are_raised(self):
442447
"""Log-fetch errors other than a missing log group still propagate."""

0 commit comments

Comments
 (0)