3333from botocore .exceptions import ClientError
3434
3535from sagemaker .core .jumpstart import JumpStartConfig
36+ from sagemaker .core .helper .session_helper import Session
3637from sagemaker .train import ModelTrainer
37- from sagemaker .train .configs import Compute , InputData
38+ from sagemaker .train .configs import Compute
3839
3940logger = logging .getLogger (__name__ )
4041
4445ALIASED_REFERENCE_NAME = "sdk-integ-aliased-catboost"
4546PRIVATE_MODEL_NAME = "sdk-integ-private-catboost"
4647
48+ # Only these error codes represent "this restricted account is not allowed to
49+ # set up the fixture" and warrant a graceful skip. Any other ClientError is a
50+ # real failure (e.g. a service-side regression in create_hub_content_reference
51+ # or import_hub_content) and must fail loudly so the test does not silently
52+ # stop guarding the fix while CI stays green.
53+ _SKIPPABLE_SETUP_ERROR_CODES = frozenset (
54+ {
55+ "AccessDeniedException" ,
56+ "AccessForbiddenException" ,
57+ "UnauthorizedOperation" ,
58+ }
59+ )
60+
61+
62+ def _skip_if_unauthorized (e , message ):
63+ """Skip only on an expected authorization error; re-raise everything else."""
64+ if e .response .get ("Error" , {}).get ("Code" ) in _SKIPPABLE_SETUP_ERROR_CODES :
65+ pytest .skip (f"{ message } : { e } " )
66+ raise
67+
68+
69+ def _assert_reference_channels (model_trainer ):
70+ """Assert the SDK resolved a ModelReference into hub-aware training channels.
71+
72+ Resolving a ModelReference must produce a container image and attach a
73+ HubAccessConfig(hub_content_arn=...) to the model channel (defaults.py,
74+ hub_content_type == "ModelReference" branch). Asserting this on the
75+ SDK-resolved channels — rather than passing an explicit training channel to
76+ train() — is what actually guards the fix; a non-gated model would train
77+ fine even if this plumbing regressed.
78+ """
79+ assert model_trainer .training_image
80+ model_channels = [
81+ c for c in model_trainer .input_data_config if getattr (c , "channel_name" , None ) == "model"
82+ ]
83+ assert len (model_channels ) == 1
84+ hub_access_config = model_channels [0 ].data_source .s3_data_source .hub_access_config
85+ assert hub_access_config is not None
86+ assert hub_access_config .hub_content_arn
87+
88+
89+ def _assert_owned_model_channels (model_trainer ):
90+ """Assert a privately-owned Model resolved into direct (non-brokered) channels.
91+
92+ An owned Model (not a reference) must resolve to a model channel with a real
93+ S3 artifact URI and NO HubAccessConfig — the inverse of the reference case.
94+ """
95+ assert model_trainer .training_image
96+ model_channels = [
97+ c for c in model_trainer .input_data_config if getattr (c , "channel_name" , None ) == "model"
98+ ]
99+ assert len (model_channels ) == 1
100+ s3_source = model_channels [0 ].data_source .s3_data_source
101+ assert s3_source .s3_uri
102+ assert s3_source .hub_access_config is None
103+
47104
48105def _sm_client (sagemaker_session ):
49106 return sagemaker_session .boto_session .client ("sagemaker" )
@@ -66,11 +123,12 @@ def _wait_for_content(sm, hub_name, name, content_type, timeout=300, poll=10):
66123 deadline = time .time () + timeout
67124 while time .time () < deadline :
68125 try :
69- resp = sm .list_hub_contents (HubName = hub_name , HubContentType = content_type )
70- if any (
71- c ["HubContentName" ] == name and c .get ("HubContentStatus" ) == "Available"
72- for c in resp .get ("HubContentSummaries" , [])
73- ):
126+ resp = sm .describe_hub_content (
127+ HubName = hub_name ,
128+ HubContentName = name ,
129+ HubContentType = content_type ,
130+ )
131+ if resp .get ("HubContentStatus" ) == "Available" :
74132 return True
75133 except ClientError :
76134 pass
@@ -107,23 +165,8 @@ def _delete_hub(sm, hub_name):
107165 logger .warning ("Failed to delete hub %s: %s" , hub_name , e )
108166
109167
110- def _default_training_dataset (region , model_id ):
111- """Resolve the model's default training dataset S3 URI from JS metadata."""
112- from sagemaker .core .jumpstart .accessors import JumpStartModelsAccessor
113-
114- specs = JumpStartModelsAccessor .get_model_specs (region = region , model_id = model_id , version = "*" )
115- if not getattr (specs , "training_supported" , False ):
116- return None
117- key = getattr (specs , "default_training_dataset_key" , None )
118- if not key :
119- return None
120- return f"s3://jumpstart-cache-prod-{ region } /{ key } "
121-
122-
123168@pytest .fixture (scope = "module" )
124169def sagemaker_session ():
125- from sagemaker .core .helper .session_helper import Session
126-
127170 return Session ()
128171
129172
@@ -138,14 +181,14 @@ def private_hub(sagemaker_session):
138181 HubDescription = "SDK integ test JumpStart training private hub" ,
139182 )
140183 except ClientError as e :
141- pytest . skip ( f "Cannot create private hub (missing permissions?): { e } " )
184+ _skip_if_unauthorized ( e , "Cannot create private hub (missing permissions?)" )
142185
143186 for _ in range (30 ):
144187 if sm .describe_hub (HubName = hub_name )["HubStatus" ] == "InService" :
145188 break
146189 time .sleep (2 )
147190 else :
148- pytest .skip (f"Hub { hub_name } did not reach InService" )
191+ pytest .fail (f"Hub { hub_name } did not reach InService" )
149192
150193 yield hub_name
151194 _delete_hub (sm , hub_name )
@@ -198,13 +241,11 @@ def test_jumpstart_train_from_private_hub_reference(private_hub, sagemaker_sessi
198241 SageMakerPublicHubContentArn = _public_model_arn (region , TRAINABLE_MODEL_ID ),
199242 )
200243 except ClientError as e :
201- pytest . skip ( f "Cannot create hub content reference: { e } " )
244+ _skip_if_unauthorized ( e , "Cannot create hub content reference" )
202245 if not _wait_for_content (sm , private_hub , TRAINABLE_MODEL_ID , "ModelReference" ):
203- pytest .skip (f"ModelReference { TRAINABLE_MODEL_ID } not available in { private_hub } " )
204-
205- dataset = _default_training_dataset (region , TRAINABLE_MODEL_ID )
206- if dataset is None :
207- pytest .skip (f"{ TRAINABLE_MODEL_ID } is not trainable / has no default dataset" )
246+ pytest .fail (
247+ f"ModelReference { TRAINABLE_MODEL_ID } did not become Available in { private_hub } "
248+ )
208249
209250 jumpstart = JumpStartConfig (model_id = TRAINABLE_MODEL_ID , hub_name = private_hub , accept_eula = True )
210251 model_trainer = ModelTrainer .from_jumpstart_config (
@@ -214,7 +255,16 @@ def test_jumpstart_train_from_private_hub_reference(private_hub, sagemaker_sessi
214255 compute = Compute (instance_type = "ml.m5.xlarge" ),
215256 sagemaker_session = sagemaker_session ,
216257 )
217- model_trainer .train (input_data_config = [InputData (channel_name = "training" , data_source = dataset )])
258+
259+ # Assert the fix's plumbing on the SDK-resolved channels before training:
260+ # resolving a ModelReference must attach a HubAccessConfig(hub_content_arn=...)
261+ # to the model channel. This is the assertion that would catch the plumbing
262+ # regressing; a non-gated model would otherwise train fine even if it broke.
263+ _assert_reference_channels (model_trainer )
264+
265+ # Train on the SDK-resolved channels (no explicit training channel), so the
266+ # hub-aware channel construction under test is actually exercised.
267+ model_trainer .train ()
218268
219269
220270def test_jumpstart_train_from_aliased_reference (private_hub , sagemaker_session ):
@@ -230,13 +280,9 @@ def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):
230280 HubContentName = ALIASED_REFERENCE_NAME ,
231281 )
232282 except ClientError as e :
233- pytest . skip ( f "Cannot create aliased hub content reference: { e } " )
283+ _skip_if_unauthorized ( e , "Cannot create aliased hub content reference" )
234284 if not _wait_for_content (sm , private_hub , ALIASED_REFERENCE_NAME , "ModelReference" ):
235- pytest .skip (f"Aliased reference { ALIASED_REFERENCE_NAME } not available" )
236-
237- dataset = _default_training_dataset (region , TRAINABLE_MODEL_ID )
238- if dataset is None :
239- pytest .skip (f"{ TRAINABLE_MODEL_ID } is not trainable / has no default dataset" )
285+ pytest .fail (f"Aliased reference { ALIASED_REFERENCE_NAME } did not become Available" )
240286
241287 jumpstart = JumpStartConfig (
242288 model_id = TRAINABLE_MODEL_ID ,
@@ -251,15 +297,21 @@ def test_jumpstart_train_from_aliased_reference(private_hub, sagemaker_session):
251297 compute = Compute (instance_type = "ml.m5.xlarge" ),
252298 sagemaker_session = sagemaker_session ,
253299 )
254- model_trainer .train (input_data_config = [InputData (channel_name = "training" , data_source = dataset )])
300+
301+ # The alias must have been threaded through resolution (not the model_id).
302+ assert model_trainer ._jumpstart_config .hub_content_name == ALIASED_REFERENCE_NAME
303+ # ...and resolving it as a ModelReference must attach the hub-aware channels.
304+ _assert_reference_channels (model_trainer )
305+
306+ # Train on the SDK-resolved channels (no explicit training channel).
307+ model_trainer .train ()
255308
256309
257310def test_jumpstart_train_from_private_owned_model (private_hub , sagemaker_session ):
258311 """Train from a privately-owned Model authored directly into a private hub
259312 (content-type Model, not a ModelReference). Exercises the document.py
260313 fallback-to-Model resolution probe."""
261314 sm = _sm_client (sagemaker_session )
262- region = _region (sagemaker_session )
263315
264316 # Author a private Model by importing a trainable public model's document
265317 # into the private hub as content-type Model.
@@ -270,7 +322,7 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
270322 HubContentName = TRAINABLE_MODEL_ID ,
271323 )
272324 except ClientError as e :
273- pytest . skip ( f "Cannot read public model document: { e } " )
325+ _skip_if_unauthorized ( e , "Cannot read public model document" )
274326
275327 try :
276328 sm .import_hub_content (
@@ -285,13 +337,12 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
285337 HubContentSearchKeywords = public .get ("HubContentSearchKeywords" , []),
286338 )
287339 except ClientError as e :
288- pytest .skip (f"import_hub_content for a private Model not permitted/supported: { e } " )
340+ # Only skip if the account is simply not allowed to author a private
341+ # Model. Any other failure is a real regression in the owned-Model path
342+ # (the core case this fix enables) and must fail loudly.
343+ _skip_if_unauthorized (e , "import_hub_content for a private Model not permitted" )
289344 if not _wait_for_content (sm , private_hub , PRIVATE_MODEL_NAME , "Model" ):
290- pytest .skip (f"Private Model { PRIVATE_MODEL_NAME } not available in { private_hub } " )
291-
292- dataset = _default_training_dataset (region , TRAINABLE_MODEL_ID )
293- if dataset is None :
294- pytest .skip (f"{ TRAINABLE_MODEL_ID } is not trainable / has no default dataset" )
345+ pytest .fail (f"Private Model { PRIVATE_MODEL_NAME } did not become Available in { private_hub } " )
295346
296347 jumpstart = JumpStartConfig (
297348 model_id = TRAINABLE_MODEL_ID ,
@@ -306,4 +357,9 @@ def test_jumpstart_train_from_private_owned_model(private_hub, sagemaker_session
306357 compute = Compute (instance_type = "ml.m5.xlarge" ),
307358 sagemaker_session = sagemaker_session ,
308359 )
309- model_trainer .train (input_data_config = [InputData (channel_name = "training" , data_source = dataset )])
360+
361+ # Owned Model (fallback-to-Model probe): direct S3 artifact, no HubAccessConfig.
362+ _assert_owned_model_channels (model_trainer )
363+
364+ # Train on the SDK-resolved channels (no explicit training channel).
365+ model_trainer .train ()
0 commit comments