Skip to content

Commit cfb8c26

Browse files
committed
fix(local): Align local mode with the SageMaker service
Bring local mode behavior in line with the hosted SageMaker service: - Runtime invoke_endpoint now returns ResponseMetadata and InvokedProductionVariant, matching the boto3 runtime response. - LocalSagemakerClient gains a describe_user_profile passthrough so Studio role resolution works instead of raising AttributeError. - MultiRecordStrategy buffers bytes or str based on the record type, so local batch transform handles binary inputs. - Processing jobs no longer require an IAM role in local mode, where the role is never used. - Local pipeline execution state is isolated per instance and per session instead of living on class-level or client-level state. - Local pipeline execution ids are 12-char uppercase alphanumerics, matching the service format and downstream name limits. This ports #5283 (merged to master-v2, by aviruthen) to the V3 codebase; see #5269. Fixes #3348 Fixes #4417 Fixes #4996 Fixes #5562 Fixes #5572 Fixes #5604 --- X-AI-Prompt: Fix S-effort PySDK V3 bugs, local-mode theme X-AI-Tool: Kiro
1 parent 7ac2708 commit cfb8c26

10 files changed

Lines changed: 312 additions & 52 deletions

File tree

‎sagemaker-core/src/sagemaker/core/local/data.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -357,15 +357,19 @@ def pad(self, file, size=6):
357357
Returns:
358358
generator of records
359359
"""
360-
buffer = ""
360+
buffer = None
361361
for element in self.splitter.split(file):
362+
if buffer is None:
363+
# Match the buffer type to the record type so binary inputs
364+
# concatenate correctly instead of raising a TypeError.
365+
buffer = b"" if isinstance(element, bytes) else ""
362366
if _payload_size_within_limit(buffer + element, size):
363367
buffer += element
364368
else:
365369
tmp = buffer
366370
buffer = element
367371
yield tmp
368-
if _validate_payload_size(buffer, size):
372+
if buffer is not None and _validate_payload_size(buffer, size):
369373
yield buffer
370374

371375

‎sagemaker-core/src/sagemaker/core/local/local_session.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,25 @@ def __init__(self, sagemaker_session=None):
8686
"""
8787
self.sagemaker_session = sagemaker_session or LocalSession()
8888

89+
def describe_user_profile(self, DomainId, UserProfileName, **kwargs):
90+
"""Passes ``describe_user_profile`` through to the real SageMaker client.
91+
92+
Local mode does not model user profiles, but Studio role resolution calls
93+
this during session setup. Delegating to a real boto client keeps that path
94+
working instead of raising ``AttributeError``.
95+
96+
Args:
97+
DomainId (str): The domain ID the user profile belongs to.
98+
UserProfileName (str): The name of the user profile to describe.
99+
**kwargs: Additional keyword arguments forwarded to the boto client.
100+
101+
Returns: (dict) DescribeUserProfile response.
102+
"""
103+
boto_client = self.sagemaker_session.boto_session.client("sagemaker")
104+
return boto_client.describe_user_profile(
105+
DomainId=DomainId, UserProfileName=UserProfileName, **kwargs
106+
)
107+
89108
@_telemetry_emitter(Feature.LOCAL_MODE, "local_session.create_processing_job")
90109
def create_processing_job(
91110
self,
@@ -533,7 +552,25 @@ def invoke_endpoint(
533552
Body = Body.encode("utf-8")
534553
r = self.http.request("POST", url, body=Body, preload_content=False, headers=headers)
535554

536-
return {"Body": r, "ContentType": Accept}
555+
# Mirror the response shape of the real SageMaker runtime client so the same
556+
# code works in local mode and against a hosted endpoint.
557+
try:
558+
response_headers = dict(r.headers)
559+
except (TypeError, ValueError):
560+
response_headers = {}
561+
request_id = response_headers.get("x-amzn-RequestId", "local-request-id")
562+
status_code = getattr(r, "status", None)
563+
return {
564+
"Body": r,
565+
"ContentType": response_headers.get("Content-type", Accept),
566+
"InvokedProductionVariant": TargetVariant or "AllTraffic",
567+
"ResponseMetadata": {
568+
"RequestId": request_id,
569+
"HTTPStatusCode": status_code,
570+
"HTTPHeaders": response_headers,
571+
"RetryAttempts": 0,
572+
},
573+
}
537574

538575

539576
class LocalSession(Session):

‎sagemaker-core/src/sagemaker/core/processing.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,11 +278,12 @@ def __init__(
278278
self.role = resolve_value_from_config(
279279
role, PROCESSING_JOB_ROLE_ARN_PATH, sagemaker_session=self.sagemaker_session
280280
)
281-
if not self.role:
281+
if not self.role and self.instance_type not in ("local", "local_gpu"):
282282
# Originally IAM role was a required parameter.
283283
# Now we marked that as Optional because we can fetch it from SageMakerConfig
284284
# Because of marking that parameter as optional, we should validate if it is None, even
285-
# after fetching the config.
285+
# after fetching the config. In Local Mode the role is never used
286+
# (LocalSagemakerClient.create_processing_job discards it), so it is not required.
286287
raise ValueError("An AWS IAM role is required to create a Processing job.")
287288

288289
self.env = resolve_value_from_config(
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
14+
"""Unit tests for sagemaker.core.local.data batch strategies"""
15+
16+
from __future__ import absolute_import
17+
18+
from sagemaker.core.local.data import MultiRecordStrategy
19+
20+
21+
class _ListSplitter:
22+
"""A minimal Splitter stand-in that yields pre-set records."""
23+
24+
def __init__(self, records):
25+
self._records = records
26+
27+
def split(self, file): # pylint: disable=unused-argument
28+
for record in self._records:
29+
yield record
30+
31+
32+
def test_multi_record_strategy_pad_bytes():
33+
"""Binary records must not raise TypeError from str + bytes concatenation."""
34+
records = [b"abc", b"def", b"ghi"]
35+
strategy = MultiRecordStrategy(_ListSplitter(records))
36+
37+
result = list(strategy.pad("dummy", size=6))
38+
39+
assert result == [b"abcdefghi"]
40+
assert all(isinstance(chunk, bytes) for chunk in result)
41+
42+
43+
def test_multi_record_strategy_pad_str():
44+
"""Text records continue to be grouped as strings."""
45+
records = ["abc", "def", "ghi"]
46+
strategy = MultiRecordStrategy(_ListSplitter(records))
47+
48+
result = list(strategy.pad("dummy", size=6))
49+
50+
assert result == ["abcdefghi"]
51+
assert all(isinstance(chunk, str) for chunk in result)
52+
53+
54+
def test_multi_record_strategy_pad_empty():
55+
"""An empty split yields nothing instead of raising."""
56+
strategy = MultiRecordStrategy(_ListSplitter([]))
57+
58+
result = list(strategy.pad("dummy", size=6))
59+
60+
assert result == []

‎sagemaker-core/tests/unit/local/test_local_session.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,78 @@ def test_invoke_endpoint_with_string_body(self, mock_pool_manager_class, mock_ge
372372
# String should be encoded to bytes
373373
assert isinstance(body, bytes)
374374

375+
@patch("sagemaker.core.local.local_session.get_docker_host")
376+
@patch("urllib3.PoolManager")
377+
def test_invoke_endpoint_response_shape(self, mock_pool_manager_class, mock_get_host):
378+
"""Response mirrors the real runtime client (ResponseMetadata + variant)."""
379+
mock_get_host.return_value = "localhost"
380+
381+
mock_pool = Mock()
382+
mock_response = Mock()
383+
mock_response.status = 200
384+
mock_response.headers = {
385+
"Content-type": "application/json",
386+
"x-amzn-RequestId": "req-abc-123",
387+
}
388+
mock_pool.request.return_value = mock_response
389+
mock_pool_manager_class.return_value = mock_pool
390+
391+
client = LocalSagemakerRuntimeClient()
392+
393+
response = client.invoke_endpoint(
394+
Body=b"test data",
395+
EndpointName="test-endpoint",
396+
Accept="application/json",
397+
TargetVariant="variant1",
398+
)
399+
400+
assert response["Body"] == mock_response
401+
assert response["ContentType"] == "application/json"
402+
assert response["InvokedProductionVariant"] == "variant1"
403+
metadata = response["ResponseMetadata"]
404+
assert metadata["HTTPStatusCode"] == 200
405+
assert metadata["RequestId"] == "req-abc-123"
406+
assert metadata["HTTPHeaders"]["Content-type"] == "application/json"
407+
assert metadata["RetryAttempts"] == 0
408+
409+
@patch("sagemaker.core.local.local_session.get_docker_host")
410+
@patch("urllib3.PoolManager")
411+
def test_invoke_endpoint_default_variant(self, mock_pool_manager_class, mock_get_host):
412+
"""Without TargetVariant the response reports the default variant name."""
413+
mock_get_host.return_value = "localhost"
414+
415+
mock_pool = Mock()
416+
mock_response = Mock()
417+
mock_response.status = 200
418+
mock_response.headers = {}
419+
mock_pool.request.return_value = mock_response
420+
mock_pool_manager_class.return_value = mock_pool
421+
422+
client = LocalSagemakerRuntimeClient()
423+
424+
response = client.invoke_endpoint(Body=b"data", EndpointName="ep")
425+
426+
assert response["InvokedProductionVariant"] == "AllTraffic"
427+
assert response["ResponseMetadata"]["RequestId"] == "local-request-id"
428+
429+
def test_describe_user_profile_passthrough(self):
430+
"""describe_user_profile delegates to the real boto SageMaker client."""
431+
mock_session = Mock()
432+
mock_boto_client = Mock()
433+
mock_boto_client.describe_user_profile.return_value = {
434+
"UserSettings": {"ExecutionRole": "arn:aws:iam::123456789012:role/Studio"}
435+
}
436+
mock_session.boto_session.client.return_value = mock_boto_client
437+
438+
client = LocalSagemakerClient(sagemaker_session=mock_session)
439+
result = client.describe_user_profile(DomainId="d-123", UserProfileName="user-a")
440+
441+
mock_session.boto_session.client.assert_called_once_with("sagemaker")
442+
mock_boto_client.describe_user_profile.assert_called_once_with(
443+
DomainId="d-123", UserProfileName="user-a"
444+
)
445+
assert result["UserSettings"]["ExecutionRole"] == ("arn:aws:iam::123456789012:role/Studio")
446+
375447

376448
class TestLocalSession:
377449
"""Test cases for LocalSession"""

‎sagemaker-core/tests/unit/test_processing.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2292,3 +2292,38 @@ def test_sparkjar_processor_forwards_instance_preferences(self, mock_session):
22922292
sagemaker_session=mock_session,
22932293
)
22942294
assert processor.instance_preferences == self._PREFS
2295+
2296+
2297+
class TestProcessorLocalModeRole:
2298+
def test_role_not_required_for_local_instance(self, mock_session):
2299+
"""A role is optional when instance_type is local (role is unused locally)."""
2300+
processor = Processor(
2301+
image_uri="test-image:latest",
2302+
instance_count=1,
2303+
instance_type="local",
2304+
sagemaker_session=mock_session,
2305+
)
2306+
assert processor.role is None
2307+
assert processor.instance_type == "local"
2308+
2309+
def test_role_not_required_for_local_gpu_instance(self, mock_session):
2310+
"""A role is optional when instance_type is local_gpu."""
2311+
processor = Processor(
2312+
image_uri="test-image:latest",
2313+
instance_count=1,
2314+
instance_type="local_gpu",
2315+
sagemaker_session=mock_session,
2316+
)
2317+
assert processor.role is None
2318+
2319+
def test_role_still_required_for_managed_instance(self, mock_session):
2320+
"""A missing role still raises for non-local instance types."""
2321+
with pytest.raises(
2322+
ValueError, match="An AWS IAM role is required to create a Processing job."
2323+
):
2324+
Processor(
2325+
image_uri="test-image:latest",
2326+
instance_count=1,
2327+
instance_type="ml.m5.xlarge",
2328+
sagemaker_session=mock_session,
2329+
)

‎sagemaker-mlops/src/sagemaker/mlops/local/local_pipeline_session.py‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,10 @@ def __init__(self, *args, **kwargs):
4747
Accepts the same arguments as LocalSession.
4848
"""
4949
super().__init__(*args, **kwargs)
50-
# Add pipeline storage to the sagemaker_client
51-
if not hasattr(self.sagemaker_client, "_pipelines"):
52-
self.sagemaker_client._pipelines = {}
50+
# Own the local pipeline registry on the session rather than mutating the
51+
# sagemaker_client. Attaching state to the client leaks across sessions that
52+
# share a client and risks colliding with real client attributes.
53+
self._local_pipelines = {}
5354

5455
@_telemetry_emitter(Feature.LOCAL_MODE, "local_pipeline_session.create_pipeline")
5556
def create_pipeline(
@@ -69,7 +70,7 @@ def create_pipeline(
6970
pipeline_description=pipeline_description,
7071
local_session=self,
7172
)
72-
self.sagemaker_client._pipelines[pipeline.name] = local_pipeline
73+
self._local_pipelines[pipeline.name] = local_pipeline
7374
return {"PipelineArn": pipeline.name}
7475

7576
def update_pipeline(
@@ -84,19 +85,17 @@ def update_pipeline(
8485
Returns:
8586
Pipeline metadata (PipelineArn)
8687
"""
87-
if pipeline.name not in self.sagemaker_client._pipelines:
88+
if pipeline.name not in self._local_pipelines:
8889
error_response = {
8990
"Error": {
9091
"Code": "ResourceNotFound",
9192
"Message": "Pipeline {} does not exist".format(pipeline.name),
9293
}
9394
}
9495
raise ClientError(error_response, "update_pipeline")
95-
self.sagemaker_client._pipelines[pipeline.name].pipeline_description = pipeline_description
96-
self.sagemaker_client._pipelines[pipeline.name].pipeline = pipeline
97-
self.sagemaker_client._pipelines[pipeline.name].last_modified_time = (
98-
datetime.now().timestamp()
99-
)
96+
self._local_pipelines[pipeline.name].pipeline_description = pipeline_description
97+
self._local_pipelines[pipeline.name].pipeline = pipeline
98+
self._local_pipelines[pipeline.name].last_modified_time = datetime.now().timestamp()
10099
return {"PipelineArn": pipeline.name}
101100

102101
def describe_pipeline(self, PipelineName):
@@ -108,15 +107,15 @@ def describe_pipeline(self, PipelineName):
108107
Returns:
109108
Pipeline metadata (PipelineArn, PipelineDefinition, LastModifiedTime, etc)
110109
"""
111-
if PipelineName not in self.sagemaker_client._pipelines:
110+
if PipelineName not in self._local_pipelines:
112111
error_response = {
113112
"Error": {
114113
"Code": "ResourceNotFound",
115114
"Message": "Pipeline {} does not exist".format(PipelineName),
116115
}
117116
}
118117
raise ClientError(error_response, "describe_pipeline")
119-
return self.sagemaker_client._pipelines[PipelineName].describe()
118+
return self._local_pipelines[PipelineName].describe()
120119

121120
def delete_pipeline(self, PipelineName):
122121
"""Delete the local pipeline.
@@ -127,8 +126,8 @@ def delete_pipeline(self, PipelineName):
127126
Returns:
128127
Pipeline metadata (PipelineArn)
129128
"""
130-
if PipelineName in self.sagemaker_client._pipelines:
131-
del self.sagemaker_client._pipelines[PipelineName]
129+
if PipelineName in self._local_pipelines:
130+
del self._local_pipelines[PipelineName]
132131
return {"PipelineArn": PipelineName}
133132

134133
def start_pipeline_execution(self, PipelineName, **kwargs):
@@ -144,12 +143,12 @@ def start_pipeline_execution(self, PipelineName, **kwargs):
144143
logger.warning("Parallelism configuration is not supported in local mode.")
145144
if "SelectiveExecutionConfig" in kwargs:
146145
raise ValueError("SelectiveExecutionConfig is not supported in local mode.")
147-
if PipelineName not in self.sagemaker_client._pipelines:
146+
if PipelineName not in self._local_pipelines:
148147
error_response = {
149148
"Error": {
150149
"Code": "ResourceNotFound",
151150
"Message": "Pipeline {} does not exist".format(PipelineName),
152151
}
153152
}
154153
raise ClientError(error_response, "start_pipeline_execution")
155-
return self.sagemaker_client._pipelines[PipelineName].start(**kwargs)
154+
return self._local_pipelines[PipelineName].start(**kwargs)

0 commit comments

Comments
 (0)