Skip to content

Commit b4b0b80

Browse files
authored
test(serve): run the recommender integ tests on ml.g6 and ml.g7 instances (#6557)
test_recommendation_workflow_end_to_end's recommendation job, and the endpoint the test deploys from its top recommendation, now use ml.g7.2xlarge instead of ml.g5.2xlarge. The JumpStart model, which the benchmark test deploys and the recommendation test builds, now uses ml.g6.2xlarge: JumpStart has no ml.g7 image configuration for it, and ml.g6.2xlarge gets the same serving image as ml.g5.2xlarge.
1 parent e8edd7e commit b4b0b80

1 file changed

Lines changed: 9 additions & 4 deletions

File tree

‎sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
# serves /v1/chat/completions, matching the OpenAI api_standard the
4444
# synthetic Workload sends.
4545
MODEL_ID = "huggingface-llm-qwen2-1-5b-instruct"
46-
INSTANCE_TYPE = "ml.g5.2xlarge"
46+
MODEL_INSTANCE_TYPE = "ml.g6.2xlarge"
47+
RECOMMENDATION_INSTANCE_TYPE = "ml.g7.2xlarge"
4748
WORKLOAD_TOKENIZER = "gpt2"
4849
WORKLOAD_REQUEST_COUNT = 10
4950
WORKLOAD_CONCURRENCY = 1
@@ -63,7 +64,7 @@ def _build_synthetic_workload():
6364

6465

6566
def _build_jumpstart_model_builder(role_arn):
66-
compute = Compute(instance_type=INSTANCE_TYPE)
67+
compute = Compute(instance_type=MODEL_INSTANCE_TYPE)
6768
jumpstart_config = JumpStartConfig(model_id=MODEL_ID)
6869
return ModelBuilder.from_jumpstart_config(
6970
jumpstart_config=jumpstart_config, compute=compute, role_arn=role_arn
@@ -152,7 +153,7 @@ def test_recommendation_workflow_end_to_end():
152153
recommendation_job = model_builder.generate_deployment_recommendations(
153154
workload=_build_synthetic_workload(),
154155
performance_target="throughput",
155-
instance_types=[INSTANCE_TYPE],
156+
instance_types=[RECOMMENDATION_INSTANCE_TYPE],
156157
advanced_optimization=False,
157158
framework="LMI",
158159
role_arn=role,
@@ -197,10 +198,14 @@ def test_recommendation_workflow_end_to_end():
197198
wait=True,
198199
)
199200
logger.info(f"Recommendation endpoint deployed: {rec_endpoint.endpoint_name}")
201+
rec_model = Model.get(model_name=rec_model_name)
200202
assert (
201203
rec_endpoint.endpoint_status == "InService"
202204
), f"Endpoint did not reach InService: {rec_endpoint.endpoint_status}"
203-
rec_model = Model.get(model_name=rec_model_name)
205+
variants = EndpointConfig.get(
206+
endpoint_config_name=rec_endpoint_config_name
207+
).production_variants
208+
assert [variant.instance_type for variant in variants] == [RECOMMENDATION_INSTANCE_TYPE]
204209

205210
finally:
206211
if rec_endpoint and rec_model:

0 commit comments

Comments
 (0)