Skip to content

Commit e1859f1

Browse files
committed
test(serve): run the recommender integ tests on ml.g6 and ml.g7 instances
test_recommendation_workflow_end_to_end's recommendation job, and the endpoint the test deploys from its top recommendation, now use ml.g7.2xlarge instead of ml.g5.2xlarge. The JumpStart model, which the benchmark test deploys and the recommendation test builds, now uses ml.g6.2xlarge: JumpStart has no ml.g7 image configuration for it, and ml.g6.2xlarge gets the same serving image as ml.g5.2xlarge.
1 parent 02cd0dd commit e1859f1

1 file changed

Lines changed: 8 additions & 3 deletions

File tree

‎sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@
4343
# serves /v1/chat/completions, matching the OpenAI api_standard the
4444
# synthetic Workload sends.
4545
MODEL_ID = "huggingface-llm-qwen2-1-5b-instruct"
46-
INSTANCE_TYPE = "ml.g5.2xlarge"
46+
MODEL_INSTANCE_TYPE = "ml.g6.2xlarge"
47+
RECOMMENDATION_INSTANCE_TYPE = "ml.g7.2xlarge"
4748
WORKLOAD_TOKENIZER = "gpt2"
4849
WORKLOAD_REQUEST_COUNT = 10
4950
WORKLOAD_CONCURRENCY = 1
@@ -63,7 +64,7 @@ def _build_synthetic_workload():
6364

6465

6566
def _build_jumpstart_model_builder(role_arn):
66-
compute = Compute(instance_type=INSTANCE_TYPE)
67+
compute = Compute(instance_type=MODEL_INSTANCE_TYPE)
6768
jumpstart_config = JumpStartConfig(model_id=MODEL_ID)
6869
return ModelBuilder.from_jumpstart_config(
6970
jumpstart_config=jumpstart_config, compute=compute, role_arn=role_arn
@@ -152,7 +153,7 @@ def test_recommendation_workflow_end_to_end():
152153
recommendation_job = model_builder.generate_deployment_recommendations(
153154
workload=_build_synthetic_workload(),
154155
performance_target="throughput",
155-
instance_types=[INSTANCE_TYPE],
156+
instance_types=[RECOMMENDATION_INSTANCE_TYPE],
156157
advanced_optimization=False,
157158
framework="LMI",
158159
role_arn=role,
@@ -200,6 +201,10 @@ def test_recommendation_workflow_end_to_end():
200201
assert (
201202
rec_endpoint.endpoint_status == "InService"
202203
), f"Endpoint did not reach InService: {rec_endpoint.endpoint_status}"
204+
variants = EndpointConfig.get(
205+
endpoint_config_name=rec_endpoint_config_name
206+
).production_variants
207+
assert [variant.instance_type for variant in variants] == [RECOMMENDATION_INSTANCE_TYPE]
203208
rec_model = Model.get(model_name=rec_model_name)
204209

205210
finally:

0 commit comments

Comments
 (0)