diff --git a/tests/integ/sagemaker/jumpstart/model/test_jumpstart_model.py b/tests/integ/sagemaker/jumpstart/model/test_jumpstart_model.py index 4cfd67e8c9..9da067e99e 100644 --- a/tests/integ/sagemaker/jumpstart/model/test_jumpstart_model.py +++ b/tests/integ/sagemaker/jumpstart/model/test_jumpstart_model.py @@ -167,6 +167,7 @@ def test_jumpstart_gated_model_neuron(setup): assert response is not None +@x_fail_if_ice def test_jumpstart_gated_model(setup): model_id = "meta-textgeneration-llama-2-7b" diff --git a/tests/integ/sagemaker/jumpstart/private_hub/estimator/test_jumpstart_private_hub_estimator.py b/tests/integ/sagemaker/jumpstart/private_hub/estimator/test_jumpstart_private_hub_estimator.py index b94bf4f217..7dfe53238d 100644 --- a/tests/integ/sagemaker/jumpstart/private_hub/estimator/test_jumpstart_private_hub_estimator.py +++ b/tests/integ/sagemaker/jumpstart/private_hub/estimator/test_jumpstart_private_hub_estimator.py @@ -30,6 +30,7 @@ get_sm_session, get_training_dataset_for_model_and_version, add_model_references_to_hub, + x_fail_if_ice, ) MAX_INIT_TIME_SECONDS = 5 @@ -135,6 +136,7 @@ def test_jumpstart_hub_estimator_with_session(setup, add_model_references): @pytest.mark.slow_test +@x_fail_if_ice def test_jumpstart_hub_gated_estimator_with_eula(setup, add_model_references): model_id, model_version = "meta-textgeneration-llama-2-7b", "*" diff --git a/tests/integ/sagemaker/jumpstart/private_hub/model/test_jumpstart_private_hub_model.py b/tests/integ/sagemaker/jumpstart/private_hub/model/test_jumpstart_private_hub_model.py index 214c70aaea..fee5bdc32b 100644 --- a/tests/integ/sagemaker/jumpstart/private_hub/model/test_jumpstart_private_hub_model.py +++ b/tests/integ/sagemaker/jumpstart/private_hub/model/test_jumpstart_private_hub_model.py @@ -31,6 +31,7 @@ from tests.integ.sagemaker.jumpstart.utils import ( get_sm_session, add_model_references_to_hub, + x_fail_if_ice, ) MAX_INIT_TIME_SECONDS = 5 @@ -91,6 +92,7 @@ def test_jumpstart_hub_model_with_default_session(setup, add_model_references): @pytest.mark.slow_test +@x_fail_if_ice def test_jumpstart_hub_gated_model(setup, add_model_references): model_id = "meta-textgeneration-llama-3-2-1b" diff --git a/tests/integ/sagemaker/lineage/conftest.py b/tests/integ/sagemaker/lineage/conftest.py index a211089699..2a3dce1a91 100644 --- a/tests/integ/sagemaker/lineage/conftest.py +++ b/tests/integ/sagemaker/lineage/conftest.py @@ -662,6 +662,12 @@ def static_endpoint_context(sagemaker_session, static_pipeline_execution_arn): ) endpoint_arn = get_endpoint_arn_from_static_pipeline(sagemaker_session) + if endpoint_arn is None: + raise Exception( + f"Static endpoint {STATIC_ENDPOINT_NAME} does not exist and could not be deployed " + f"from static pipeline execution {static_pipeline_execution_arn}." + ) + contexts = sagemaker_session.sagemaker_client.list_contexts(SourceUri=endpoint_arn)[ "ContextSummaries" ] @@ -888,11 +894,36 @@ def _deploy_static_endpoint(execution_arn, sagemaker_session): model_package.deploy(1, "ml.m5.xlarge", endpoint_name=STATIC_ENDPOINT_NAME) time.sleep(120) except ClientError as e: - if e.response["Error"]["Code"] == "ValidationException": - print(f"Endpoint {STATIC_ENDPOINT_NAME} already exists. Continuing.") - pass - else: + if e.response["Error"]["Code"] != "ValidationException": raise (e) + if get_endpoint_arn_from_static_pipeline(sagemaker_session) is not None: + print(f"Endpoint {STATIC_ENDPOINT_NAME} already exists. Continuing.") + return + # The endpoint is gone but an endpoint config with the same name outlived it (e.g. the + # endpoint was deleted out of band), so deploy() failed on CreateEndpointConfig without + # creating an endpoint. Recreate the endpoint from the existing endpoint config. + print( + f"Endpoint {STATIC_ENDPOINT_NAME} is missing but its endpoint config exists. " + "Recreating the endpoint from the existing endpoint config." + ) + _create_static_endpoint_from_existing_config(sagemaker_session) + + +def _create_static_endpoint_from_existing_config(sagemaker_session): + try: + sagemaker_session.sagemaker_client.create_endpoint( + EndpointName=STATIC_ENDPOINT_NAME, EndpointConfigName=STATIC_ENDPOINT_NAME + ) + except ClientError as e: + # Another test worker may have recreated the endpoint concurrently; anything else + # (e.g. the endpoint config is missing too) is a real failure. + if ( + e.response["Error"]["Code"] != "ValidationException" + or get_endpoint_arn_from_static_pipeline(sagemaker_session) is None + ): + raise + sagemaker_session.wait_for_endpoint(STATIC_ENDPOINT_NAME) + time.sleep(120) @pytest.fixture diff --git a/tests/integ/sagemaker/serve/test_base_model_builder_deploy.py b/tests/integ/sagemaker/serve/test_base_model_builder_deploy.py index 56f8962c2b..e252b641ee 100644 --- a/tests/integ/sagemaker/serve/test_base_model_builder_deploy.py +++ b/tests/integ/sagemaker/serve/test_base_model_builder_deploy.py @@ -14,7 +14,6 @@ import os import uuid -from typing import Generator import numpy as np import pandas as pd @@ -39,43 +38,43 @@ from sagemaker.serve import InferenceSpec, SchemaBuilder from sagemaker.serve.builder.model_builder import ModelBuilder from sagemaker.serverless.serverless_inference_config import ServerlessInferenceConfig -from tests.integ.utils import cleanup_model_resources -@pytest.fixture(autouse=True) -def cleanup_endpoints(mb_sagemaker_session) -> Generator[None, None, None]: - """Clean up any existing endpoints before and after tests.""" - sagemaker_client = mb_sagemaker_session.sagemaker_client +def _cleanup_deployment(model_builder, endpoint_name): + """Best-effort delete of the endpoint, endpoint config and model one test created. - # Pre-test cleanup + Only touch resources created under ``endpoint_name``. The integ suites run concurrently + in a shared account, so sweeping every endpoint there deletes endpoints that other running + tests and long-lived static fixtures (e.g. the lineage tests' static endpoint) depend on. + """ + sagemaker_client = model_builder.sagemaker_session.sagemaker_client + endpoint_config_name = endpoint_name try: - endpoints = sagemaker_client.list_endpoints() - for endpoint in endpoints["Endpoints"]: - try: - sagemaker_client.delete_endpoint(EndpointName=endpoint["EndpointName"]) - sagemaker_client.delete_endpoint_config( - EndpointConfigName=endpoint["EndpointConfigName"] - ) - except Exception as e: - print(f"Error cleaning up endpoint {endpoint['EndpointName']}: {e}") - except Exception as e: - print(f"Error listing endpoints: {e}") - - yield - - # Post-test cleanup - try: - endpoints = sagemaker_client.list_endpoints() - for endpoint in endpoints["Endpoints"]: - try: - sagemaker_client.delete_endpoint(EndpointName=endpoint["EndpointName"]) - sagemaker_client.delete_endpoint_config( - EndpointConfigName=endpoint["EndpointConfigName"] - ) - except Exception as e: - print(f"Error cleaning up endpoint {endpoint['EndpointName']}: {e}") - except Exception as e: - print(f"Error listing endpoints: {e}") + endpoint_config_name = sagemaker_client.describe_endpoint(EndpointName=endpoint_name)[ + "EndpointConfigName" + ] + except Exception: # noqa: BLE001 -- the endpoint may never have been created + pass + + deletions = [ + ("endpoint", lambda: sagemaker_client.delete_endpoint(EndpointName=endpoint_name)), + ( + "endpoint config", + lambda: sagemaker_client.delete_endpoint_config( + EndpointConfigName=endpoint_config_name + ), + ), + ( + "model", + lambda: sagemaker_client.delete_model(ModelName=model_builder.built_model.name), + ), + ] + # Each deletion runs independently, so one failure does not leave the rest behind. + for resource, delete in deletions: + try: + delete() + except Exception as e: # noqa: BLE001 -- best-effort cleanup + print(f"Error cleaning up {resource} for endpoint {endpoint_name}: {e}") @pytest.fixture(scope="module") @@ -184,44 +183,41 @@ def invoke(self, input_object: object, model: object): def test_real_time_deployment(xgboost_model_builder): - real_time_predictor = xgboost_model_builder.deploy( - endpoint_name=f"test-{uuid.uuid1().hex}", initial_instance_count=1 - ) + endpoint_name = f"test-{uuid.uuid1().hex}" + try: + real_time_predictor = xgboost_model_builder.deploy( + endpoint_name=endpoint_name, initial_instance_count=1 + ) - assert real_time_predictor is not None - cleanup_model_resources( - sagemaker_session=xgboost_model_builder.sagemaker_session, - model_name=xgboost_model_builder.built_model.name, - endpoint_name=xgboost_model_builder.built_model.endpoint_name, - ) + assert real_time_predictor is not None + finally: + _cleanup_deployment(xgboost_model_builder, endpoint_name) def test_serverless_deployment(xgboost_model_builder): - serverless_predictor = xgboost_model_builder.deploy( - endpoint_name=f"test1-{uuid.uuid1().hex}", inference_config=ServerlessInferenceConfig() - ) + endpoint_name = f"test1-{uuid.uuid1().hex}" + try: + serverless_predictor = xgboost_model_builder.deploy( + endpoint_name=endpoint_name, inference_config=ServerlessInferenceConfig() + ) - assert serverless_predictor is not None - cleanup_model_resources( - sagemaker_session=xgboost_model_builder.sagemaker_session, - model_name=xgboost_model_builder.built_model.name, - endpoint_name=xgboost_model_builder.built_model.endpoint_name, - ) + assert serverless_predictor is not None + finally: + _cleanup_deployment(xgboost_model_builder, endpoint_name) def test_async_deployment(xgboost_model_builder, mb_sagemaker_session): - async_predictor = xgboost_model_builder.deploy( - endpoint_name=f"test2-{uuid.uuid1().hex}", - inference_config=AsyncInferenceConfig( - output_path=s3_path_join( - "s3://", mb_sagemaker_session.default_bucket(), "async_inference/output" - ) - ), - ) + endpoint_name = f"test2-{uuid.uuid1().hex}" + try: + async_predictor = xgboost_model_builder.deploy( + endpoint_name=endpoint_name, + inference_config=AsyncInferenceConfig( + output_path=s3_path_join( + "s3://", mb_sagemaker_session.default_bucket(), "async_inference/output" + ) + ), + ) - assert async_predictor is not None - cleanup_model_resources( - sagemaker_session=xgboost_model_builder.sagemaker_session, - model_name=xgboost_model_builder.built_model.name, - endpoint_name=xgboost_model_builder.built_model.endpoint_name, - ) + assert async_predictor is not None + finally: + _cleanup_deployment(xgboost_model_builder, endpoint_name) diff --git a/tests/integ/sagemaker/workflow/test_deployment_steps.py b/tests/integ/sagemaker/workflow/test_deployment_steps.py index 8c27ac33fd..705d67e5c9 100644 --- a/tests/integ/sagemaker/workflow/test_deployment_steps.py +++ b/tests/integ/sagemaker/workflow/test_deployment_steps.py @@ -27,6 +27,7 @@ import os import time import uuid +from datetime import datetime, timedelta, timezone import pytest @@ -39,6 +40,10 @@ INSTANCE_TYPE = "ml.m5.xlarge" EXECUTION_TIMEOUT_SECONDS = 45 * 60 POLL_SECONDS = 30 +# Largest page size the List* APIs accept. +LIST_PAGE_SIZE = 100 +# Margin on the CreationTimeAfter filter to absorb clock skew with the service. +CREATION_TIME_MARGIN = timedelta(minutes=10) # sagemaker_session and pipeline_session come from tests/conftest.py, and role from @@ -50,6 +55,7 @@ def test_deployment_steps_execute_end_to_end(sagemaker_session, pipeline_session, role): """Chained EndpointConfig -> Endpoint -> InferenceComponent pipeline run.""" stamp = uuid.uuid4().hex[:8] + created_after = datetime.now(timezone.utc) - CREATION_TIME_MARGIN config_name = f"integ-deploy-cfg-{stamp}" endpoint_name = f"integ-deploy-ep-{stamp}" component_name = f"integ-deploy-ic-{stamp}" @@ -155,14 +161,21 @@ def test_deployment_steps_execute_end_to_end(sagemaker_session, pipeline_session # Resolve the actual server-side names (the steps suffix them). actual_config = _resolve( - sm_client.list_endpoint_configs(NameContains=stamp)["EndpointConfigs"], + _list_all(sm_client, "list_endpoint_configs", "EndpointConfigs", stamp, created_after), "EndpointConfigName", ) actual_endpoint = _resolve( - sm_client.list_endpoints(NameContains=stamp)["Endpoints"], "EndpointName" + _list_all(sm_client, "list_endpoints", "Endpoints", stamp, created_after), + "EndpointName", ) actual_component = _resolve( - sm_client.list_inference_components(NameContains=stamp)["InferenceComponents"], + _list_all( + sm_client, + "list_inference_components", + "InferenceComponents", + stamp, + created_after, + ), "InferenceComponentName", ) @@ -180,7 +193,26 @@ def test_deployment_steps_execute_end_to_end(sagemaker_session, pipeline_session ) assert component_desc["EndpointName"] == actual_endpoint finally: - _cleanup(sagemaker_session, sm_client, stamp, pipeline, model_name) + _cleanup(sagemaker_session, sm_client, stamp, created_after, pipeline, model_name) + + +def _list_all(sm_client, operation, result_key, stamp, created_after): + """Return every resource whose name contains ``stamp``, across all result pages. + + SageMaker applies ``NameContains`` to each page of results rather than before paging, + so a single call can return an empty page (plus a NextToken) even though a matching + resource exists. That happens whenever more than a page of other resources were created + after this run's, which is routine while the rest of the integ suite runs in parallel in + the same account, so always walk every page. ``CreationTimeAfter`` and the larger page + size keep the number of pages small. + """ + paginator = sm_client.get_paginator(operation) + pages = paginator.paginate( + NameContains=stamp, + CreationTimeAfter=created_after, + PaginationConfig={"PageSize": LIST_PAGE_SIZE}, + ) + return [item for page in pages for item in page[result_key]] def _resolve(items, key): @@ -189,21 +221,25 @@ def _resolve(items, key): return items[0][key] -def _cleanup(sagemaker_session, sm_client, stamp, pipeline, model_name): +def _cleanup(sagemaker_session, sm_client, stamp, created_after, pipeline, model_name): """Delete every resource this run created, in dependency order.""" - for component in sm_client.list_inference_components(NameContains=stamp)["InferenceComponents"]: + for component in _list_all( + sm_client, "list_inference_components", "InferenceComponents", stamp, created_after + ): name = component["InferenceComponentName"] try: sm_client.delete_inference_component(InferenceComponentName=name) _wait_component_deleted(sm_client, name) except Exception: # noqa: BLE001 -- best-effort cleanup pass - for endpoint in sm_client.list_endpoints(NameContains=stamp)["Endpoints"]: + for endpoint in _list_all(sm_client, "list_endpoints", "Endpoints", stamp, created_after): try: sm_client.delete_endpoint(EndpointName=endpoint["EndpointName"]) except Exception: # noqa: BLE001 pass - for config in sm_client.list_endpoint_configs(NameContains=stamp)["EndpointConfigs"]: + for config in _list_all( + sm_client, "list_endpoint_configs", "EndpointConfigs", stamp, created_after + ): try: sm_client.delete_endpoint_config(EndpointConfigName=config["EndpointConfigName"]) except Exception: # noqa: BLE001 diff --git a/tests/integ/test_huggingface_torch_distributed.py b/tests/integ/test_huggingface_torch_distributed.py index d741a7ee01..591f986dea 100644 --- a/tests/integ/test_huggingface_torch_distributed.py +++ b/tests/integ/test_huggingface_torch_distributed.py @@ -16,9 +16,11 @@ import pytest from sagemaker.huggingface import HuggingFace from tests.integ import DATA_DIR, TRAINING_DEFAULT_TIMEOUT_MINUTES, timeout +from tests.integ.sagemaker.jumpstart.utils import fit_estimator_with_capacity_xfail, x_fail_if_ice @pytest.mark.slow_test +@x_fail_if_ice def test_huggingface_torch_distributed_g5_glue( sagemaker_session, huggingface_training_latest_version, @@ -49,4 +51,7 @@ def test_huggingface_torch_distributed_g5_glue( sagemaker_session=sagemaker_session, disable_profiler=True, ) - estimator.fit() + # ml.g5.4xlarge capacity is often unavailable, and a job waiting for capacity never + # fails on its own: it just sits in "Pending" until the timeout above kills the test. + # Fail fast with a CapacityError instead, which x_fail_if_ice turns into an xfail. + fit_estimator_with_capacity_xfail(estimator, inputs=None) diff --git a/tests/integ/test_inference_recommender.py b/tests/integ/test_inference_recommender.py index 4337ceb5da..d300a4f571 100644 --- a/tests/integ/test_inference_recommender.py +++ b/tests/integ/test_inference_recommender.py @@ -13,6 +13,7 @@ from __future__ import absolute_import import os +import random import time import pytest @@ -42,16 +43,34 @@ IR_SKLEARN_FRAMEWORK_VERSION = "1.0-1" -def retry_and_back_off(right_size_fn): - tot_retries = 3 - retries = 1 - while retries <= tot_retries: +def retry_and_back_off(right_size_fn, cleanup_fn=None, tot_retries=3): + """Call ``right_size_fn`` and retry it with back-off while the service throttles it. + + The module-scoped fixtures below start Inference Recommender jobs at the same time from + parallel test workers, and CreateInferenceRecommendationsJob is rate limited. + + Args: + right_size_fn: Zero-argument callable that runs ``right_size()``. It must be a + callable rather than the result of ``right_size()``: an already evaluated call + raises before this function can catch anything. + cleanup_fn: Optional zero-argument callable run before each retry. ``right_size()`` + creates the SageMaker model of an unregistered model before it creates the job, + so that model has to be deleted before a retry can create it again. + tot_retries: Total number of attempts. + """ + for attempt in range(1, tot_retries + 1): try: - return right_size_fn + return right_size_fn() except ClientError as e: - if e.response["Error"]["Code"] == "ThrottlingException": - retries += 1 - time.sleep(5 * retries) + if e.response["Error"]["Code"] != "ThrottlingException" or attempt == tot_retries: + raise + logger.warning( + "right_size() throttled on attempt %d/%d, retrying", attempt, tot_retries + ) + if cleanup_fn is not None: + cleanup_fn() + # Jitter so the parallel workers that were throttled together do not retry together. + time.sleep(5 * attempt + random.uniform(0, 5)) @pytest.fixture(scope="module") @@ -83,7 +102,7 @@ def default_right_sized_model(sagemaker_session, cpu_instance_type): return ( retry_and_back_off( - sklearn_model_package.right_size( + lambda: sklearn_model_package.right_size( job_name=ir_job_name, sample_payload_url=payload_data, supported_content_types=IR_SKLEARN_CONTENT_TYPE, @@ -102,6 +121,7 @@ def default_right_sized_model(sagemaker_session, cpu_instance_type): sagemaker_session.sagemaker_client.delete_model_package_group( ModelPackageGroupName=model_package_group_name ) + raise @pytest.fixture(scope="module") @@ -150,7 +170,7 @@ def advanced_right_sized_model(sagemaker_session, cpu_instance_type): return ( retry_and_back_off( - sklearn_model_package.right_size( + lambda: sklearn_model_package.right_size( sample_payload_url=payload_data, supported_content_types=IR_SKLEARN_CONTENT_TYPE, framework=IR_SKLEARN_FRAMEWORK, @@ -195,19 +215,23 @@ def default_right_sized_unregistered_model(sagemaker_session, cpu_instance_type) return ( retry_and_back_off( - sklearn_model.right_size( + lambda: sklearn_model.right_size( job_name=ir_job_name, sample_payload_url=payload_data, supported_content_types=IR_SKLEARN_CONTENT_TYPE, supported_instance_types=[cpu_instance_type], framework=IR_SKLEARN_FRAMEWORK, log_level="Quiet", - ) + ), + cleanup_fn=lambda: sagemaker_session.delete_model( + model_name=sklearn_model.name + ), ), ir_job_name, ) except Exception: sagemaker_session.delete_model(model_name=sklearn_model.name) + raise @pytest.fixture(scope="module") @@ -246,7 +270,7 @@ def advanced_right_sized_unregistered_model(sagemaker_session, cpu_instance_type ] return retry_and_back_off( - sklearn_model.right_size( + lambda: sklearn_model.right_size( sample_payload_url=payload_data, supported_content_types=IR_SKLEARN_CONTENT_TYPE, framework=IR_SKLEARN_FRAMEWORK, @@ -258,11 +282,13 @@ def advanced_right_sized_unregistered_model(sagemaker_session, cpu_instance_type max_tests=5, max_parallel_tests=5, log_level="Quiet", - ) + ), + cleanup_fn=lambda: sagemaker_session.delete_model(model_name=sklearn_model.name), ) except Exception: sagemaker_session.delete_model(model_name=sklearn_model.name) + raise @pytest.fixture(scope="module") @@ -289,19 +315,21 @@ def default_right_sized_unregistered_base_model(sagemaker_session, cpu_instance_ return ( retry_and_back_off( - model.right_size( + lambda: model.right_size( job_name=ir_job_name, sample_payload_url=payload_data, supported_content_types=IR_SKLEARN_CONTENT_TYPE, supported_instance_types=[cpu_instance_type], framework=IR_SKLEARN_FRAMEWORK, log_level="Quiet", - ) + ), + cleanup_fn=lambda: sagemaker_session.delete_model(model_name=model.name), ), ir_job_name, ) except Exception: sagemaker_session.delete_model(model_name=model.name) + raise @pytest.fixture(scope="module") @@ -336,6 +364,7 @@ def test_default_right_size_and_deploy_registered_model_sklearn( endpoint_name = unique_name_from_base("test-ir-right-size-default-sklearn") right_size_model_package, model_package_group_name, ir_job_name = default_right_sized_model + predictor = None with timeout(minutes=45): try: right_size_model_package.predictor_cls = SKLearnPredictor @@ -347,8 +376,9 @@ def test_default_right_size_and_deploy_registered_model_sklearn( assert inference is not None assert 26 == len(inference) finally: - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() @pytest.mark.slow_test @@ -359,6 +389,7 @@ def test_default_right_size_and_deploy_unregistered_model_sklearn( endpoint_name = unique_name_from_base("test-ir-right-size-default-unregistered-sklearn") right_size_model, ir_job_name = default_right_sized_unregistered_model + predictor = None with timeout(minutes=45): try: right_size_model.predictor_cls = SKLearnPredictor @@ -370,8 +401,9 @@ def test_default_right_size_and_deploy_unregistered_model_sklearn( assert inference is not None assert 26 == len(inference) finally: - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() @pytest.mark.slow_test @@ -382,6 +414,7 @@ def test_default_right_size_and_deploy_unregistered_base_model( endpoint_name = unique_name_from_base("test-ir-right-size-default-unregistered-base") right_size_model, ir_job_name = default_right_sized_unregistered_base_model + predictor = None with timeout(minutes=45): try: right_size_model.predictor_cls = SKLearnPredictor @@ -393,8 +426,9 @@ def test_default_right_size_and_deploy_unregistered_base_model( assert inference is not None assert 26 == len(inference) finally: - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() @pytest.mark.slow_test @@ -405,6 +439,7 @@ def test_advanced_right_size_and_deploy_unregistered_model_sklearn( endpoint_name = unique_name_from_base("test-ir-right-size-advanced-sklearn") right_size_model = advanced_right_sized_unregistered_model + predictor = None with timeout(minutes=45): try: right_size_model.predictor_cls = SKLearnPredictor @@ -416,8 +451,9 @@ def test_advanced_right_size_and_deploy_unregistered_model_sklearn( assert inference is not None assert 26 == len(inference) finally: - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() @pytest.mark.skip(reason="Skipping this test class for now") @@ -429,6 +465,7 @@ def test_advanced_right_size_and_deploy_registered_model_sklearn( endpoint_name = unique_name_from_base("test-ir-right-size-advanced-sklearn") right_size_model_package, model_package_group_name = advanced_right_sized_model + predictor = None with timeout(minutes=45): try: right_size_model_package.predictor_cls = SKLearnPredictor @@ -446,8 +483,9 @@ def test_advanced_right_size_and_deploy_registered_model_sklearn( sagemaker_session.sagemaker_client.delete_model_package_group( ModelPackageGroupName=model_package_group_name ) - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() # TODO when we've added support for inference_recommendation_id @@ -465,6 +503,7 @@ def test_deploy_inference_recommendation_id_with_registered_model_sklearn( rec_id = get_realtime_recommendation_id(recommendation_list=rec_res["InferenceRecommendations"]) + predictor = None with timeout(minutes=45): try: right_size_model_package.predictor_cls = SKLearnPredictor @@ -484,13 +523,15 @@ def test_deploy_inference_recommendation_id_with_registered_model_sklearn( sagemaker_session.sagemaker_client.delete_model_package_group( ModelPackageGroupName=model_package_group_name ) - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() @pytest.mark.slow_test @pytest.mark.flaky(reruns=3, reruns_delay=2) def test_deploy_deployment_recommendation_id_with_model(created_base_model, sagemaker_session): + predictor = None with timeout(minutes=20): try: deployment_recommendation = poll_for_deployment_recommendation( @@ -518,8 +559,9 @@ def test_deploy_deployment_recommendation_id_with_model(created_base_model, sage assert inference is not None assert 26 == len(inference) finally: - predictor.delete_model() - predictor.delete_endpoint() + if predictor is not None: + predictor.delete_model() + predictor.delete_endpoint() def poll_for_deployment_recommendation(created_base_model, sagemaker_session): diff --git a/tests/integ/test_sagemaker_config.py b/tests/integ/test_sagemaker_config.py index 8cc8d50053..7f09b71ba7 100644 --- a/tests/integ/test_sagemaker_config.py +++ b/tests/integ/test_sagemaker_config.py @@ -333,6 +333,16 @@ def test_sagemaker_config_cross_context_injection( endpoint_config_1.pop(key) endpoint_config_2.pop(key) + # Service-side defaults that the SDK and the sagemaker_config never set, and that the + # service has changed over time: MetricsConfig is now returned by default, and the + # default VolumeSizeInGB depends on the instance type. Comparing them would make this + # test fail whenever the service changes a default, without testing config injection. + for description in [endpoint_1, endpoint_2, endpoint_config_1, endpoint_config_2]: + description.pop("MetricsConfig", None) + for description in [endpoint_config_1, endpoint_config_2]: + for variant in description["ProductionVariants"]: + variant.pop("VolumeSizeInGB", None) + for key in ["ResponseMetadata"]: model_tags.pop(key) endpoint_1_tags.pop(key) @@ -385,7 +395,6 @@ def test_sagemaker_config_cross_context_injection( "InitialInstanceCount": 1, "InstanceType": alternative_cpu_instance_type, "InitialVariantWeight": 1.0, - "VolumeSizeInGB": 4, } ], "DataCaptureConfig": { @@ -413,7 +422,6 @@ def test_sagemaker_config_cross_context_injection( "InitialInstanceCount": 1, "InstanceType": cpu_instance_type, "InitialVariantWeight": 1.0, - "VolumeSizeInGB": 16, } ], "EnableNetworkIsolation": False,