test: xfail CI Health V2 tests on GPU capacity shortages (#6396)

* test: xfail CI Health V2 tests on GPU capacity shortages

test_predict_stream deploys a falcon-7b LMI endpoint on ml.g5.2xlarge
in its module fixture. When us-west-2 has no capacity, the deploy fails
with InsufficientInstanceCapacity, the test errors at setup, and
pytest-rerunfailures retries the ~30 minute deploy several times. Mark
it as an expected failure instead, matching the JumpStart integ tests.

test_model_package_arn_jumpstart_model already imports x_fail_if_ice but
was missing the decorator, so ml.g4dn.2xlarge capacity shortages failed
the slow-tests-v2 job. Apply the decorator.

* fix: stop live-logging endpoint wait from hanging on early failure

session._live_logging_deploy_done returned None whenever the endpoint's
CloudWatch log group was missing, even after the endpoint had left
Creating. The log group only exists once a container starts, so an
endpoint that fails before provisioning (InsufficientInstanceCapacity,
a missing model) was polled forever. ModelBuilder enables endpoint
logging by default, so test_serverless_deployment hung the PR integ
build until the CodeBuild timeout. Return the describe response once
the endpoint is no longer in progress, as sagemaker-core already does.

Also fix the integ flakes from the same build:
- test_base_model_builder_deploy: use a unique model name per fixture so
  one xdist worker's cleanup cannot delete the model another worker's
  endpoint is being created from.
- create_and_execute_pipeline: raise the default wait from 400s to
  1000s; two sequential function steps took ~470s.
- test_jumpstart_transfer_learning_estimator_class: xfail on
  InsufficientInstanceCapacity for ml.g4dn.xlarge.

Author: lucasjia-aws

src/sagemaker/session.py

@@ -8965,6 +8965,12 @@ def _live_logging_deploy_done(sagemaker_client, endpoint_name, paginator, pagina
             return desc
     except ClientError as e:
         if e.response["Error"]["Code"] == "ResourceNotFoundException":
+            # The log group only exists once a container has started, so an endpoint
+            # that fails before any instance is provisioned (e.g. on
+            # InsufficientInstanceCapacity, or a missing model) never gets one. A missing
+            # log group must not keep an already finished deployment waiting forever.
+            if stop:
+                return desc
             LOGGER.debug("Waiting for endpoint log group to appear")
             return None
         raise e

tests/integ/sagemaker/jumpstart/model/test_jumpstart_model.py

@@ -112,6 +112,7 @@ def test_prepacked_jumpstart_model(setup):
     tests.integ.test_region() not in GATED_INFERENCE_MODEL_PACKAGE_SUPPORTED_REGIONS,
     reason=f"JumpStart model package inference models unavailable in {tests.integ.test_region()}.",
 )
+@x_fail_if_ice
 def test_model_package_arn_jumpstart_model(setup):
 
     model_id = "meta-textgeneration-llama-2-7b"

tests/integ/sagemaker/jumpstart/script_mode_class/test_transfer_learning.py

@@ -46,9 +46,11 @@
     EndpointInvoker,
     get_sm_session,
     get_training_dataset_for_model_and_version,
+    x_fail_if_ice,
 )
 
 
+@x_fail_if_ice
 def test_jumpstart_transfer_learning_estimator_class(setup):
 
     model_id, model_version = "huggingface-spc-bert-base-cased", "1.2.3"

tests/integ/sagemaker/serve/test_base_model_builder_deploy.py

@@ -170,7 +170,11 @@ def invoke(self, input_object: object, model: object):
     training_job.wait()
 
     xgboost_model_builder = ModelBuilder(
-        name="ModelBuilderTest",
+        # Unique per fixture instance: xdist runs this module's tests on separate
+        # workers, and each test's cleanup deletes the model. A shared fixed name
+        # let one worker delete the model another worker's endpoint was being
+        # created from ("Model ... does not exist").
+        name=f"ModelBuilderTest-{uuid.uuid4().hex[:8]}",
         model_path=training_job.model_artifacts.s3_model_artifacts,
         role_arn=role,
         inference_spec=XGBoostSpec(),

tests/integ/sagemaker/workflow/helpers.py

@@ -38,7 +38,9 @@ def create_and_execute_pipeline(
     step_status,
     step_result_type=None,
     step_result_value=None,
-    wait_duration=400,  # seconds
+    # Two sequential function steps routinely take ~8 minutes when each step's
+    # instance spends ~3 minutes in Starting, so 400s was too tight.
+    wait_duration=1000,  # seconds
     selective_execution_config=None,
 ):
     create_arn = None

tests/integ/test_predict_stream.py

@@ -20,6 +20,7 @@
 import tests.integ.timeout
 
 from sagemaker import image_uris
+from sagemaker.exceptions import UnexpectedStatusException
 from sagemaker.iterators import LineIterator
 from sagemaker.model import Model
 from sagemaker.predictor import Predictor
@@ -56,12 +57,21 @@ def endpoint_name(sagemaker_session):
             name=lmi_endpoint_name,  # model name
             role=ROLE,
         )
-        lmi_model.deploy(
-            INSTANCE_COUNT,
-            INSTANCE_TYPE,
-            endpoint_name=lmi_endpoint_name,
-            container_startup_health_check_timeout=900,
-        )
+        try:
+            lmi_model.deploy(
+                INSTANCE_COUNT,
+                INSTANCE_TYPE,
+                endpoint_name=lmi_endpoint_name,
+                container_startup_health_check_timeout=900,
+            )
+        except UnexpectedStatusException as e:
+            # Insufficient ml.g5.2xlarge capacity is a transient, region-level AWS
+            # condition, not a SDK defect. Mark it as an expected failure so the
+            # canary doesn't go red, and so pytest-rerunfailures doesn't retry a
+            # ~30 minute deploy that is unlikely to get capacity on retry.
+            if "InsufficientInstanceCapacity" in str(e):
+                pytest.xfail(str(e))
+            raise
         yield lmi_endpoint_name
 
 

tests/unit/test_session.py

@@ -7374,3 +7374,57 @@ def test_get_most_recently_created_approved_model_package(sagemaker_session):
         model_package.model_package_arn
         == "arn:aws:sagemaker:us-west-2:123456789012:model-package/model-version/3"
     )
+
+
+def _log_group_not_found():
+    return ClientError(
+        {"Error": {"Code": "ResourceNotFoundException", "Message": "log group not found"}},
+        "FilterLogEvents",
+    )
+
+
+@patch("sagemaker.session.time.sleep")
+def test_live_logging_deploy_done_returns_failed_endpoint_without_log_group(sleep):
+    # An endpoint that fails before any container starts never gets a log group.
+    # The wait must stop instead of polling for the log group forever.
+    desc = {"EndpointStatus": "Failed", "FailureReason": "InsufficientInstanceCapacity"}
+    sagemaker_client = Mock(describe_endpoint=Mock(return_value=desc))
+    paginator = Mock(paginate=Mock(side_effect=_log_group_not_found()))
+
+    result = sagemaker.session._live_logging_deploy_done(
+        sagemaker_client, "my-endpoint", paginator, {}, poll=5
+    )
+
+    assert result == desc
+    sleep.assert_called_once_with(5)
+
+
+@patch("sagemaker.session.time.sleep")
+def test_live_logging_deploy_done_keeps_waiting_for_creating_endpoint_without_log_group(sleep):
+    sagemaker_client = Mock(describe_endpoint=Mock(return_value={"EndpointStatus": "Creating"}))
+    paginator = Mock(paginate=Mock(side_effect=_log_group_not_found()))
+
+    result = sagemaker.session._live_logging_deploy_done(
+        sagemaker_client, "my-endpoint", paginator, {}, poll=5
+    )
+
+    assert result is None
+    sleep.assert_not_called()
+
+
+def test_wait_for_endpoint_live_logging_raises_on_failed_endpoint_without_log_group(
+    sagemaker_session,
+):
+    failed = {"EndpointStatus": "Failed", "FailureReason": "Model does not exist."}
+    # Bound the polls so a regression fails the test instead of hanging it.
+    sagemaker_session.sagemaker_client.describe_endpoint = Mock(
+        side_effect=[failed] * 3 + [AssertionError("wait_for_endpoint kept polling")]
+    )
+    logs_client = Mock()
+    logs_client.get_paginator.return_value.paginate.side_effect = _log_group_not_found()
+    sagemaker_session.boto_session.client = Mock(return_value=logs_client)
+
+    with patch("sagemaker.session._has_permission_for_live_logging", return_value=True):
+        with patch("sagemaker.session.time.sleep"):
+            with pytest.raises(exceptions.UnexpectedStatusException, match="Model does not exist"):
+                sagemaker_session.wait_for_endpoint("my-endpoint", live_logging=True)