Skip to content

Commit c9ec795

Browse files
mohamedzeidan2021Mohamed Zeidan
andauthored
fix: keep the Transformer's configured session when waiting on or stopping a transform job (#6509)
Co-authored-by: Mohamed Zeidan <zeidmo@amazon.com>
1 parent 6e99f9b commit c9ec795

2 files changed

Lines changed: 264 additions & 7 deletions

File tree

‎sagemaker-core/src/sagemaker/core/transformer.py‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ def __init__(
155155
self.base_transform_job_name = base_transform_job_name
156156
self._current_job_name = None
157157
self.latest_transform_job = None
158+
self._latest_transform_job_is_local = False
158159
self._reset_output_path = False
159160

160161
self.sagemaker_session = sagemaker_session or Session()
@@ -412,10 +413,21 @@ def submit(request):
412413
transformed = transform_util(serialized_request, "CreateTransformJobRequest")
413414
# Remove tags from transformed dict as TransformJob resource doesn't accept it
414415
transformed.pop("tags", None)
415-
self.latest_transform_job = TransformJob(**transformed)
416+
# Submission went through the configured session, so bind that session and
417+
# region to the resource: a session-less TransformJob builds its client from
418+
# the process default session, so wait()/stop() poll the wrong account or
419+
# region and fail with ResourceNotFound, NoRegionError or an authorization
420+
# error even though the job was created correctly (#6503).
421+
self.latest_transform_job = TransformJob(**transformed)._set_client_context(
422+
session=self.sagemaker_session.boto_session,
423+
region=self.sagemaker_session.boto_region_name,
424+
)
425+
# A local transform job runs to completion inside create_transform_job and
426+
# exists only in the local session, so it must never be polled over AWS.
427+
self._latest_transform_job_is_local = local_mode
416428

417429
if wait:
418-
self.latest_transform_job.wait(logs=logs)
430+
self.wait(logs=logs)
419431

420432
def delete_model(self):
421433
"""Delete the corresponding SageMaker model for this Transformer."""
@@ -462,13 +474,26 @@ def _retrieve_image_uri(self):
462474
)
463475

464476
def wait(self, logs=True):
465-
"""Placeholder docstring"""
477+
"""Wait for the latest transform job to reach a terminal state."""
466478
self._ensure_last_transform_job()
479+
if self._latest_transform_job_is_local:
480+
logger.info(
481+
"Local transform job %s already ran to completion; nothing to wait for.",
482+
self.latest_transform_job.get_name(),
483+
)
484+
return
467485
self.latest_transform_job.wait(logs=logs)
468486

469487
def stop_transform_job(self, wait=True):
470488
"""Stop latest running batch transform job."""
471489
self._ensure_last_transform_job()
490+
if self._latest_transform_job_is_local:
491+
logger.warning(
492+
"Transform job %s ran locally and has already finished; there is nothing "
493+
"to stop.",
494+
self.latest_transform_job.get_name(),
495+
)
496+
return
472497
self.latest_transform_job.stop()
473498
if wait:
474499
self.latest_transform_job.wait()
@@ -495,17 +520,20 @@ def attach(cls, transform_job_name, sagemaker_session=None):
495520
"""
496521
sagemaker_session = sagemaker_session or Session()
497522

523+
# TransformJob.get() builds its client from a boto3 session, so pass the
524+
# session's boto_session rather than the sagemaker Session itself, and keep
525+
# the resource that describe returned instead of describing it a second time.
498526
transform_job = TransformJob.get(
499-
transform_job_name=transform_job_name, session=sagemaker_session
527+
transform_job_name=transform_job_name,
528+
session=sagemaker_session.boto_session,
529+
region=sagemaker_session.boto_region_name,
500530
)
501531
if not transform_job:
502532
raise ValueError(f"Transform job {transform_job_name} not found")
503533
job_details = transform_job.__dict__
504534
init_params = cls._prepare_init_params_from_job_description(job_details)
505535
transformer = cls(sagemaker_session=sagemaker_session, **init_params)
506-
transformer.latest_transform_job = TransformJob.get(
507-
transform_job_name=init_params["base_transform_job_name"], session=sagemaker_session
508-
)
536+
transformer.latest_transform_job = transform_job
509537

510538
return transformer
511539

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Transformer keeps its configured session after submission (issue #6503).
14+
15+
``transform()`` submitted through the configured ``sagemaker_session`` and then
16+
built ``TransformJob(**transformed)`` with no session, so ``wait()``, ``stop()``
17+
and the log stream fell back to the process default boto session: the job was
18+
created in the target account and the wait failed against the default one.
19+
Local mode was worse -- the wait could poll AWS for a job that only ever
20+
existed in the local session.
21+
"""
22+
23+
from unittest.mock import MagicMock, Mock, patch
24+
25+
import boto3
26+
import pytest
27+
28+
from sagemaker.core.resources import Base, TransformJob
29+
from sagemaker.core.shapes import TransformOutput, TransformResources
30+
from sagemaker.core.transformer import Transformer
31+
from sagemaker.core.utils.utils import SageMakerClient
32+
33+
REGION = "us-east-2"
34+
DEFAULT_REGION = "us-east-1"
35+
MODEL_NAME = "transformer-session-propagation-model"
36+
JOB_NAME = "transformer-session-propagation-job"
37+
INPUT_URI = "s3://target-bucket/input/"
38+
OUTPUT_URI = "s3://target-bucket/output/"
39+
40+
# The session the caller configures (target account) and the process default one.
41+
TARGET_BOTO_SESSION = boto3.Session(
42+
aws_access_key_id="AKIDTARGET", aws_secret_access_key="secret", region_name=REGION
43+
)
44+
DEFAULT_BOTO_SESSION = boto3.Session(
45+
aws_access_key_id="AKIDDEFAULT", aws_secret_access_key="secret", region_name=DEFAULT_REGION
46+
)
47+
48+
49+
@pytest.fixture(autouse=True)
50+
def _reset_client_cache():
51+
SageMakerClient.reset()
52+
yield
53+
SageMakerClient.reset()
54+
55+
56+
def _configured_session(local_mode=False):
57+
session = MagicMock()
58+
session.boto_session = TARGET_BOTO_SESSION
59+
session.boto_region_name = REGION
60+
session.local_mode = local_mode
61+
session.default_bucket.return_value = "target-bucket"
62+
session.default_bucket_prefix = None
63+
session.sagemaker_config = {}
64+
# Run the submit callback inline; the sagemaker_client is a mock, so no AWS call.
65+
session._intercept_create_request.side_effect = lambda request, submit, name: (
66+
submit(request) if submit else None
67+
)
68+
return session
69+
70+
71+
def _described_job():
72+
"""A TransformJob resource shaped like a DescribeTransformJob response."""
73+
return TransformJob(
74+
transform_job_name=JOB_NAME,
75+
model_name=MODEL_NAME,
76+
transform_resources=TransformResources(instance_type="ml.m5.large", instance_count=1),
77+
transform_output=TransformOutput(s3_output_path=OUTPUT_URI),
78+
)
79+
80+
81+
def _transformer(session):
82+
return Transformer(
83+
model_name=MODEL_NAME,
84+
instance_count=1,
85+
instance_type="ml.m5.large",
86+
output_path=OUTPUT_URI,
87+
sagemaker_session=session,
88+
)
89+
90+
91+
@pytest.fixture
92+
def recorded_clients():
93+
"""Record the (session, region) every SageMaker client request is built with."""
94+
calls = []
95+
client = MagicMock()
96+
client.describe_transform_job.return_value = {
97+
"TransformJobName": JOB_NAME,
98+
"TransformJobStatus": "Completed",
99+
}
100+
101+
def fake(session=None, region_name=None, service_name="sagemaker"):
102+
calls.append((session, region_name))
103+
return client
104+
105+
with patch.object(Base, "get_sagemaker_client", side_effect=fake):
106+
yield calls, client
107+
108+
109+
class TestRemoteTransformBindsConfiguredSession:
110+
def test_transform_binds_session_and_region_to_the_resource(self):
111+
transformer = _transformer(_configured_session())
112+
113+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
114+
115+
job = transformer.latest_transform_job
116+
assert job._session is TARGET_BOTO_SESSION
117+
assert job._region == REGION
118+
119+
def test_waiting_inside_transform_uses_the_configured_session(self, recorded_clients):
120+
calls, _ = recorded_clients
121+
transformer = _transformer(_configured_session())
122+
123+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=True, logs=False)
124+
125+
assert calls, "the wait made no client request"
126+
assert all(call == (TARGET_BOTO_SESSION, REGION) for call in calls), calls
127+
128+
def test_wait_after_transform_uses_the_configured_session(self, recorded_clients):
129+
calls, _ = recorded_clients
130+
transformer = _transformer(_configured_session())
131+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
132+
calls.clear()
133+
134+
transformer.wait(logs=False)
135+
136+
assert all(call == (TARGET_BOTO_SESSION, REGION) for call in calls), calls
137+
138+
def test_stop_transform_job_uses_the_configured_session(self, recorded_clients):
139+
calls, _ = recorded_clients
140+
transformer = _transformer(_configured_session())
141+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
142+
calls.clear()
143+
144+
transformer.stop_transform_job(wait=False)
145+
146+
assert calls, "stop made no client request"
147+
assert all(call == (TARGET_BOTO_SESSION, REGION) for call in calls), calls
148+
149+
def test_bound_client_is_signed_with_the_configured_credentials(self):
150+
"""End to end through SageMakerClient, with a default client built first."""
151+
default = SageMakerClient(session=DEFAULT_BOTO_SESSION)
152+
transformer = _transformer(_configured_session())
153+
154+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
155+
client = transformer.latest_transform_job._get_client()
156+
157+
assert client is not default.sagemaker_client
158+
assert client._request_signer._credentials.access_key == "AKIDTARGET"
159+
160+
161+
class TestLocalTransformNeverCallsAws:
162+
def test_local_wait_does_not_request_an_aws_client(self, recorded_clients):
163+
calls, _ = recorded_clients
164+
transformer = _transformer(_configured_session(local_mode=True))
165+
166+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=True, logs=False)
167+
168+
assert calls == [], f"a local transform must not poll AWS, got {calls}"
169+
170+
def test_local_stop_does_not_request_an_aws_client(self, recorded_clients):
171+
calls, _ = recorded_clients
172+
transformer = _transformer(_configured_session(local_mode=True))
173+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
174+
calls.clear()
175+
176+
transformer.stop_transform_job(wait=False)
177+
178+
assert calls == [], f"a local transform has nothing to stop in AWS, got {calls}"
179+
180+
def test_remote_transform_is_not_marked_local(self):
181+
transformer = _transformer(_configured_session())
182+
183+
transformer.transform(data=INPUT_URI, job_name=JOB_NAME, wait=False)
184+
185+
assert transformer._latest_transform_job_is_local is False
186+
187+
188+
class TestAttachUsesTheConfiguredSession:
189+
def test_attach_passes_the_boto_session_and_reuses_the_resource(self):
190+
session = _configured_session()
191+
described = _described_job()
192+
193+
with patch.object(TransformJob, "get", return_value=described) as mock_get:
194+
transformer = Transformer.attach(JOB_NAME, sagemaker_session=session)
195+
196+
mock_get.assert_called_once_with(
197+
transform_job_name=JOB_NAME,
198+
session=TARGET_BOTO_SESSION,
199+
region=REGION,
200+
)
201+
assert transformer.latest_transform_job is described
202+
203+
def test_attach_raises_when_the_job_is_missing(self):
204+
with patch.object(TransformJob, "get", return_value=None):
205+
with pytest.raises(ValueError, match="not found"):
206+
Transformer.attach(JOB_NAME, sagemaker_session=_configured_session())
207+
208+
def test_attached_transformer_is_not_marked_local(self):
209+
session = _configured_session()
210+
described = _described_job()
211+
212+
with patch.object(TransformJob, "get", return_value=described):
213+
transformer = Transformer.attach(JOB_NAME, sagemaker_session=session)
214+
215+
assert transformer._latest_transform_job_is_local is False
216+
217+
218+
class TestEnsureLastTransformJob:
219+
def test_wait_without_a_job_raises(self):
220+
transformer = _transformer(_configured_session())
221+
222+
with pytest.raises(ValueError, match="No transform job available"):
223+
transformer.wait()
224+
225+
def test_stop_without_a_job_raises(self):
226+
transformer = _transformer(Mock(local_mode=False, sagemaker_config={}))
227+
228+
with pytest.raises(ValueError, match="No transform job available"):
229+
transformer.stop_transform_job()

0 commit comments

Comments
 (0)