Skip to content

Commit 1d58ae3

Browse files
committed
test(instance-preferences): run integ tests in the standard CI account
Both integ tests were gated on account-specific environment variables that no CI configuration sets, so every CI run skipped them and the suite passed without exercising the feature. Training coverage moves to tests/integ/train/shallow, which runs on every pull request: two accepted cases (uniform and per-preference counts; submit, Describe echo, stop) and four rejected cases (instance_type, managed spot and duplicate types client-side; the five-entry cap server-side). The environment-gated deep test is removed. The processing test uses the suite's SageMakerRole and resolves its image through image_uris, as the other sagemaker-core integration tests do. Candidate types remain overridable via *_TEST_INSTANCE_TYPES for accounts whose default types lack quota; no account-specific values remain. Verified in us-west-2: all six shallow cases pass alongside the existing shallow suite, and the processing test passes with a resolved winner.
1 parent 7d0eb92 commit 1d58ae3

3 files changed

Lines changed: 183 additions & 180 deletions

File tree

‎sagemaker-core/tests/integ/processing/test_instance_preferences_processing.py‎

Lines changed: 17 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,11 @@
2424
``SelectedInstanceCount`` report the resolved winner, which must be one of
2525
the submitted preferences.
2626
27-
Configuration (environment variables; SKIPPED when unset):
28-
29-
- ``PROCESSING_INSTANCE_PREFERENCES_TEST_ROLE_ARN`` - execution role ARN
30-
- ``PROCESSING_INSTANCE_PREFERENCES_TEST_IMAGE_URI`` - processing image
31-
- ``PROCESSING_INSTANCE_PREFERENCES_TEST_INSTANCE_TYPES`` - optional comma-separated
32-
candidate types (default: ml.m5.xlarge, ml.m5.large)
33-
- ``SAGEMAKER_ENDPOINT`` - optional endpoint
34-
override.
27+
Runs in the standard integration-test account: the execution role is the
28+
suite's ``SageMakerRole`` and the image is resolved through ``image_uris``, the
29+
same way the other ``sagemaker-core`` integration tests obtain theirs.
30+
``PROCESSING_INSTANCE_PREFERENCES_TEST_INSTANCE_TYPES`` (comma-separated)
31+
overrides the candidate types for accounts where the defaults lack quota.
3532
"""
3633

3734
from __future__ import absolute_import
@@ -40,11 +37,12 @@
4037
import time
4138
import uuid
4239

43-
import pytest
40+
import boto3
4441

45-
ROLE_ARN = os.environ.get("PROCESSING_INSTANCE_PREFERENCES_TEST_ROLE_ARN")
46-
IMAGE_URI = os.environ.get("PROCESSING_INSTANCE_PREFERENCES_TEST_IMAGE_URI")
47-
ENDPOINT = os.environ.get("SAGEMAKER_ENDPOINT")
42+
from sagemaker.core import image_uris
43+
44+
ROLE = "SageMakerRole"
45+
REGION = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", "us-west-2"))
4846

4947
PREFERENCE_TYPES = [
5048
t.strip()
@@ -56,35 +54,24 @@
5654
WAIT_TIMEOUT_SECONDS = 30 * 60
5755
POLL_SECONDS = 30
5856

59-
pytestmark = pytest.mark.skipif(
60-
not (ROLE_ARN and IMAGE_URI),
61-
reason=(
62-
"Processing Instance Preferences integ test requires "
63-
"PROCESSING_INSTANCE_PREFERENCES_TEST_ROLE_ARN and "
64-
"PROCESSING_INSTANCE_PREFERENCES_TEST_IMAGE_URI"
65-
),
66-
)
67-
6857

69-
def _sagemaker_client():
70-
"""Build a boto3 SageMaker client, honoring SAGEMAKER_ENDPOINT."""
71-
import boto3
72-
73-
region = os.environ.get("AWS_REGION", "us-west-2")
74-
return boto3.client("sagemaker", region_name=region, endpoint_url=ENDPOINT)
58+
def _processing_image():
59+
return image_uris.retrieve(
60+
"sklearn", REGION, version="1.2-1", py_version="py3", instance_type=PREFERENCE_TYPES[0]
61+
)
7562

7663

7764
def test_processor_instance_preferences_e2e():
7865
"""Create a real processing job with InstancePreferences and verify the winner."""
7966
from sagemaker.core.helper.session_helper import Session
8067
from sagemaker.core.processing import Processor
8168

82-
client = _sagemaker_client()
69+
client = boto3.client("sagemaker", region_name=REGION)
8370
session = Session(sagemaker_client=client)
8471

8572
processor = Processor(
86-
role=ROLE_ARN,
87-
image_uri=IMAGE_URI,
73+
role=ROLE,
74+
image_uri=_processing_image(),
8875
instance_count=1,
8976
instance_preferences=[{"InstanceType": t} for t in PREFERENCE_TYPES],
9077
volume_size_in_gb=30,
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Shallow submission tests for ``Compute.instance_preferences``.
14+
15+
Each accepted case submits a real ``CreateTrainingJob`` carrying an ordered
16+
``InstancePreferences`` list (no top-level ``InstanceType``), asserts the
17+
service returned an ARN and echoed the list on Describe, then stops the job.
18+
Rejected cases assert the request is refused with the documented reason.
19+
20+
Which instance type wins, and how the job runs on it, is training behaviour and
21+
belongs in the deep suites.
22+
"""
23+
24+
from __future__ import absolute_import
25+
26+
import os
27+
28+
import pytest
29+
from sagemaker.core import shapes
30+
from sagemaker.core.shapes import InstancePreference
31+
from sagemaker.core.training.configs import Compute, SourceCode
32+
from sagemaker.train.model_trainer import ModelTrainer
33+
34+
from .harness import (
35+
MAX_RUNTIME_IN_SECONDS,
36+
assert_rejected,
37+
assert_submitted,
38+
cpu_image,
39+
submitted,
40+
unique_name,
41+
)
42+
43+
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "..", "data")
44+
PARAM_SCRIPT_SOURCE_DIR = os.path.join(DATA_DIR, "params_script")
45+
46+
# Two small CPU types. INSTANCE_PREFERENCES_TEST_INSTANCE_TYPES (comma-separated)
47+
# overrides them for accounts where these lack quota.
48+
PREFERENCE_TYPES = [
49+
t.strip()
50+
for t in os.environ.get(
51+
"INSTANCE_PREFERENCES_TEST_INSTANCE_TYPES", "ml.m5.large,ml.m5.xlarge"
52+
).split(",")
53+
if t.strip()
54+
]
55+
56+
57+
def _trainer(sagemaker_session, name, compute):
58+
return ModelTrainer(
59+
sagemaker_session=sagemaker_session,
60+
training_image=cpu_image(sagemaker_session),
61+
source_code=SourceCode(
62+
source_dir=PARAM_SCRIPT_SOURCE_DIR,
63+
requirements="requirements.txt",
64+
entry_script="train.py",
65+
),
66+
compute=compute,
67+
stopping_condition=shapes.StoppingCondition(max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS),
68+
base_job_name=name,
69+
)
70+
71+
72+
def _preferences(counts=None):
73+
if counts is None:
74+
return [InstancePreference(instance_type=t) for t in PREFERENCE_TYPES]
75+
return [
76+
InstancePreference(instance_type=t, instance_count=c)
77+
for t, c in zip(PREFERENCE_TYPES, counts)
78+
]
79+
80+
81+
def _echoed_types(job):
82+
job.refresh()
83+
prefs = job.resource_config.instance_preferences
84+
return [p.instance_type for p in prefs] if prefs else []
85+
86+
87+
class TestInstancePreferencesAccepted:
88+
def test_uniform_count_is_accepted_and_echoed(self, sagemaker_session):
89+
name = unique_name("shallow-ip-uniform")
90+
trainer = _trainer(
91+
sagemaker_session,
92+
name,
93+
Compute(instance_preferences=_preferences(), instance_count=1),
94+
)
95+
96+
with submitted(trainer) as job:
97+
assert_submitted(job)
98+
assert _echoed_types(job) == PREFERENCE_TYPES
99+
# The customer never set a top-level type; the service must not invent one.
100+
assert not job.resource_config.instance_type
101+
102+
def test_per_preference_counts_are_accepted_and_echoed(self, sagemaker_session):
103+
name = unique_name("shallow-ip-per-pref")
104+
trainer = _trainer(
105+
sagemaker_session,
106+
name,
107+
Compute(instance_preferences=_preferences(counts=[1, 1])),
108+
)
109+
110+
with submitted(trainer) as job:
111+
assert_submitted(job)
112+
assert _echoed_types(job) == PREFERENCE_TYPES
113+
114+
115+
class TestInstancePreferencesRejected:
116+
"""Client-side rules must fail fast: a payload that reached the service with
117+
both a fixed type and a list would be a regression in the SDK, not a
118+
behaviour to leave for the backend to catch."""
119+
120+
def test_instance_type_with_preferences_is_rejected(self, sagemaker_session):
121+
with pytest.raises(ValueError, match="mutually exclusive with instance_type"):
122+
Compute(
123+
instance_type=PREFERENCE_TYPES[0],
124+
instance_count=1,
125+
instance_preferences=_preferences(),
126+
)
127+
128+
def test_managed_spot_with_preferences_is_rejected(self, sagemaker_session):
129+
with pytest.raises(ValueError, match="mutually exclusive with managed spot training"):
130+
Compute(
131+
instance_count=1,
132+
enable_managed_spot_training=True,
133+
instance_preferences=_preferences(),
134+
)
135+
136+
def test_duplicate_types_are_rejected(self, sagemaker_session):
137+
with pytest.raises(ValueError, match="duplicate instance types"):
138+
Compute(
139+
instance_count=1,
140+
instance_preferences=[
141+
InstancePreference(instance_type=PREFERENCE_TYPES[0]),
142+
InstancePreference(instance_type=PREFERENCE_TYPES[0]),
143+
],
144+
)
145+
146+
def test_server_rejects_more_than_five_preferences(self, sagemaker_session):
147+
"""The list cap is deliberately not enforced client-side (it is a
148+
server-side tunable), so this is the one rule that must be asserted
149+
against the service."""
150+
six = [
151+
"ml.m5.large",
152+
"ml.m5.xlarge",
153+
"ml.m5.2xlarge",
154+
"ml.m4.xlarge",
155+
"ml.c5.xlarge",
156+
"ml.c5.2xlarge",
157+
]
158+
trainer = _trainer(
159+
sagemaker_session,
160+
unique_name("shallow-ip-six"),
161+
Compute(
162+
instance_preferences=[InstancePreference(instance_type=t) for t in six],
163+
instance_count=1,
164+
),
165+
)
166+
assert_rejected(trainer, ("InstancePreferences", "5", "Member must have length"))

‎sagemaker-train/tests/integ/train/test_instance_preferences.py‎

Lines changed: 0 additions & 150 deletions
This file was deleted.

0 commit comments

Comments
 (0)