Skip to content

Commit f70f6b9

Browse files
committed
test: address review feedback on private-hub ModelTrainer tests
1 parent 0c1faa3 commit f70f6b9

2 files changed

Lines changed: 246 additions & 46 deletions

File tree

‎sagemaker-train/tests/integ/jumpstart/test_jumpstart_train.py‎

Lines changed: 102 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@
3333
from botocore.exceptions import ClientError
3434

3535
from sagemaker.core.jumpstart import JumpStartConfig
36+
from sagemaker.core.helper.session_helper import Session
3637
from sagemaker.train import ModelTrainer
37-
from sagemaker.train.configs import Compute, InputData
38+
from sagemaker.train.configs import Compute
3839

3940
logger = logging.getLogger(__name__)
4041

@@ -44,6 +45,62 @@
4445
ALIASED_REFERENCE_NAME = "sdk-integ-aliased-catboost"
4546
PRIVATE_MODEL_NAME = "sdk-integ-private-catboost"
4647

48+
# Only these error codes represent "this restricted account is not allowed to
49+
# set up the fixture" and warrant a graceful skip. Any other ClientError is a
50+
# real failure (e.g. a service-side regression in create_hub_content_reference
51+
# or import_hub_content) and must fail loudly so the test does not silently
52+
# stop guarding the fix while CI stays green.
53+
_SKIPPABLE_SETUP_ERROR_CODES = frozenset(
54+
{
55+
"AccessDeniedException",
56+
"AccessForbiddenException",
57+
"UnauthorizedOperation",
58+
}
59+
)
60+
61+
62+
def _skip_if_unauthorized(e, message):
63+
"""Skip only on an expected authorization error; re-raise everything else."""
64+
if e.response.get("Error", {}).get("Code") in _SKIPPABLE_SETUP_ERROR_CODES:
65+
pytest.skip(f"{message}: {e}")
66+
raise
67+
68+
69+
def _assert_reference_channels(model_trainer):
70+
"""Assert the SDK resolved a ModelReference into hub-aware training channels.
71+
72+
Resolving a ModelReference must produce a container image and attach a
73+
HubAccessConfig(hub_content_arn=...) to the model channel (defaults.py,
74+
hub_content_type == "ModelReference" branch). Asserting this on the
75+
SDK-resolved channels — rather than passing an explicit training channel to
76+
train() — is what actually guards the fix; a non-gated model would train
77+
fine even if this plumbing regressed.
78+
"""
79+
assert model_trainer.training_image
80+
model_channels = [
81+
c for c in model_trainer.input_data_config if getattr(c, "channel_name", None) == "model"
82+
]
83+
assert len(model_channels) == 1
84+
hub_access_config = model_channels[0].data_source.s3_data_source.hub_access_config
85+
assert hub_access_config is not None
86+
assert hub_access_config.hub_content_arn
87+
88+
89+
def _assert_owned_model_channels(model_trainer):
90+
"""Assert a privately-owned Model resolved into direct (non-brokered) channels.
91+
92+
An owned Model (not a reference) must resolve to a model channel with a real
93+
S3 artifact URI and NO HubAccessConfig — the inverse of the reference case.
94+
"""
95+
assert model_trainer.training_image
96+
model_channels = [
97+
c for c in model_trainer.input_data_config if getattr(c, "channel_name", None) == "model"
98+
]
99+
assert len(model_channels) == 1
100+
s3_source = model_channels[0].data_source.s3_data_source
101+
assert s3_source.s3_uri
102+
assert s3_source.hub_access_config is None
103+
47104

48105
def _sm_client(sagemaker_session):
49106
return sagemaker_session.boto_session.client("sagemaker")
@@ -66,11 +123,12 @@ def _wait_for_content(sm, hub_name, name, content_type, timeout=300, poll=10):
66123
deadline = time.time() + timeout
67124
while time.time() < deadline:
68125
try:
69-
resp = sm.list_hub_contents(HubName=hub_name, HubContentType=content_type)
70-
if any(
71-
c["HubContentName"] == name and c.get("HubContentStatus") == "Available"
72-
for c in resp.get("HubContentSummaries", [])
73-
):
126+
resp = sm.describe_hub_content(
127+
HubName=hub_name,
128+
HubContentName=name,
129+
HubContentType=content_type,
130+
)
131+
if resp.get("HubContentStatus") == "Available":
74132
return True
75133
except ClientError:
76134
pass
@@ -107,23 +165,8 @@ def _delete_hub(sm, hub_name):
107165
logger.warning("Failed to delete hub %s: %s", hub_name, e)
108166

109167

110-
def _default_training_dataset(region, model_id):
111-
"""Resolve the model's default training dataset S3 URI from JS metadata."""
112-
from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor
113-
114-
specs = JumpStartModelsAccessor.get_model_specs(region=region, model_id=model_id, version="*")
115-
if not getattr(specs, "training_supported", False):
116-
return None
117-
key = getattr(specs, "default_training_dataset_key", None)
118-
if not key:
119-
return None
120-
return f"s3://jumpstart-cache-prod-{region}/{key}"
121-
122-
123168
@pytest.fixture(scope="module")
124169
def sagemaker_session():
125-
from sagemaker.core.helper.session_helper import Session
126-
127170
return Session()
128171

129172

@@ -138,14 +181,14 @@ def private_hub(sagemaker_session):
138181
HubDescription="SDK integ test JumpStart training private hub",
139182
)
140183
except ClientError as e:
141-
pytest.skip(f"Cannot create private hub (missing permissions?): {e}")
184+
_skip_if_unauthorized(e, "Cannot create private hub (missing permissions?)")
142185

143186
for _ in range(30):
144187
if sm.describe_hub(HubName=hub_name)["HubStatus"] == "InService":
145188
break
146189
time.sleep(2)
147190
else:
148-
pytest.skip(f"Hub {hub_name} did not reach InService")
191+
pytest.fail(f"Hub {hub_name} did not reach InService")
149192

150193
yield hub_name
151194
_delete_hub(sm, hub_name)
@@ -198,13 +241,11 @@ def test_jumpstart_train_from_private_hub_reference(private_hub, sagemaker_sessi
198241
SageMakerPublicHubContentArn=_public_model_arn(region, TRAINABLE_MODEL_ID),
199242
)
200243
except ClientError as e:
201-
pytest.skip(f"Cannot create hub content reference: {e}")
244+
_skip_if_unauthorized(e, "Cannot create hub content reference")
202245
if not _wait_for_content(sm, private_hub, TRAINABLE_MODEL_ID, "ModelReference"):
203-
pytest.skip(f"ModelReference {TRAINABLE_MODEL_ID} not available in {private_hub}")
204-
205-
dataset = _default_training_dataset(region, TRAINABLE_MODEL_ID)
206-
if dataset is None:
207-
pytest.skip(f"{TRAINABLE_MODEL_ID} is not trainable / has no default dataset")
246+
pytest.fail(
247+
f"ModelReference {TRAINABLE_MODEL_ID} did not become Available in {private_hub}"
248+
)
208249

209250
jumpstart = JumpStartConfig(model_id=TRAINABLE_MODEL_ID, hub_name=private_hub, accept_eula=True)
210251
model_trainer = ModelTrainer.from_jumpstart_config(
@@ -214,7 +255,16 @@ def test_jumpstart_train_from_private_hub_reference(private_hub, sagemaker_sessi
214255
compute=Compute(instance_type="ml.m5.xlarge"),
215256
sagemaker_session=sagemaker_session,
216257
)
217-
model_trainer.train(input_data_config=[InputData(channel_name="training", data_source=dataset)])
258+
259+
# Assert the fix's plumbing on the SDK-resolved channels before training:
260+
# resolving a ModelReference must attach a HubAccessConfig(hub_content_arn=...)
261+
# to the model channel. This is the assertion that would catch the plumbing
262+
# regressing; a non-gated model would otherwise train fine even if it broke.
263+
_assert_reference_channels(model_trainer)
264+
265+
# Train on the SDK-resolved channels (no explicit training channel), so the
266+
# hub-aware channel construction under test is actually exercised.
267+
model_trainer.train()
218268

219269

220270
def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):
@@ -230,13 +280,9 @@ def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):
230280
HubContentName=ALIASED_REFERENCE_NAME,
231281
)
232282
except ClientError as e:
233-
pytest.skip(f"Cannot create aliased hub content reference: {e}")
283+
_skip_if_unauthorized(e, "Cannot create aliased hub content reference")
234284
if not _wait_for_content(sm, private_hub, ALIASED_REFERENCE_NAME, "ModelReference"):
235-
pytest.skip(f"Aliased reference {ALIASED_REFERENCE_NAME} not available")
236-
237-
dataset = _default_training_dataset(region, TRAINABLE_MODEL_ID)
238-
if dataset is None:
239-
pytest.skip(f"{TRAINABLE_MODEL_ID} is not trainable / has no default dataset")
285+
pytest.fail(f"Aliased reference {ALIASED_REFERENCE_NAME} did not become Available")
240286

241287
jumpstart = JumpStartConfig(
242288
model_id=TRAINABLE_MODEL_ID,
@@ -251,15 +297,21 @@ def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):
251297
compute=Compute(instance_type="ml.m5.xlarge"),
252298
sagemaker_session=sagemaker_session,
253299
)
254-
model_trainer.train(input_data_config=[InputData(channel_name="training", data_source=dataset)])
300+
301+
# The alias must have been threaded through resolution (not the model_id).
302+
assert model_trainer._jumpstart_config.hub_content_name == ALIASED_REFERENCE_NAME
303+
# ...and resolving it as a ModelReference must attach the hub-aware channels.
304+
_assert_reference_channels(model_trainer)
305+
306+
# Train on the SDK-resolved channels (no explicit training channel).
307+
model_trainer.train()
255308

256309

257310
def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session):
258311
"""Train from a privately-owned Model authored directly into a private hub
259312
(content-type Model, not a ModelReference). Exercises the document.py
260313
fallback-to-Model resolution probe."""
261314
sm = _sm_client(sagemaker_session)
262-
region = _region(sagemaker_session)
263315

264316
# Author a private Model by importing a trainable public model's document
265317
# into the private hub as content-type Model.
@@ -270,7 +322,7 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
270322
HubContentName=TRAINABLE_MODEL_ID,
271323
)
272324
except ClientError as e:
273-
pytest.skip(f"Cannot read public model document: {e}")
325+
_skip_if_unauthorized(e, "Cannot read public model document")
274326

275327
try:
276328
sm.import_hub_content(
@@ -285,13 +337,12 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
285337
HubContentSearchKeywords=public.get("HubContentSearchKeywords", []),
286338
)
287339
except ClientError as e:
288-
pytest.skip(f"import_hub_content for a private Model not permitted/supported: {e}")
340+
# Only skip if the account is simply not allowed to author a private
341+
# Model. Any other failure is a real regression in the owned-Model path
342+
# (the core case this fix enables) and must fail loudly.
343+
_skip_if_unauthorized(e, "import_hub_content for a private Model not permitted")
289344
if not _wait_for_content(sm, private_hub, PRIVATE_MODEL_NAME, "Model"):
290-
pytest.skip(f"Private Model {PRIVATE_MODEL_NAME} not available in {private_hub}")
291-
292-
dataset = _default_training_dataset(region, TRAINABLE_MODEL_ID)
293-
if dataset is None:
294-
pytest.skip(f"{TRAINABLE_MODEL_ID} is not trainable / has no default dataset")
345+
pytest.fail(f"Private Model {PRIVATE_MODEL_NAME} did not become Available in {private_hub}")
295346

296347
jumpstart = JumpStartConfig(
297348
model_id=TRAINABLE_MODEL_ID,
@@ -306,4 +357,9 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
306357
compute=Compute(instance_type="ml.m5.xlarge"),
307358
sagemaker_session=sagemaker_session,
308359
)
309-
model_trainer.train(input_data_config=[InputData(channel_name="training", data_source=dataset)])
360+
361+
# Owned Model (fallback-to-Model probe): direct S3 artifact, no HubAccessConfig.
362+
_assert_owned_model_channels(model_trainer)
363+
364+
# Train on the SDK-resolved channels (no explicit training channel).
365+
model_trainer.train()

0 commit comments

Comments
 (0)