Skip to content

Commit a42d1db

Browse files
committed
fix: SageMakerClient honors the passed boto3 session and config
SageMakerClient was a process-wide singleton: whichever instance was created first was returned to every later caller, and the session passed on those later calls was silently ignored. Any code path that created a client with default-chain credentials before the user's own session was seen (bare SageMakerClient() calls, a second profile, an assumed role) then signed every control-plane call with the wrong identity. This is the residual half of #5986 and the cause of the cross-account "RoleArn: Cross-account pass role is not allowed" failure in #6069. The config argument was also computed and then discarded, so caller timeouts and retry settings never reached the boto3 clients. Replace the singleton with a cache keyed on the constructor arguments. An explicit session (or region/config) always yields a client built from exactly those arguments; SageMakerClient() with no arguments keeps returning the process default, and the first instance created still becomes that default, so the configure-once pattern used by generated stop() methods and existing callers is unchanged. Keyed entries are a small LRU (32) so a service minting a session per request does not retain every session and its four boto3 clients; the default entry is never evicted. Construction is serialized with a lock so concurrent first callers share one instance. The caller's config is merged into the client config with the SDK user-agent suffix appended rather than replacing it. SingletonMeta is left in place for CloudWatchLogsClient. Fixes #5986 Fixes #6069 --- X-AI-Prompt: Fix the SageMakerClient singleton so an explicitly passed boto3 session is honored (#5986/#6069) X-AI-Tool: Kiro
1 parent 8b43e0c commit a42d1db

2 files changed

Lines changed: 320 additions & 8 deletions

File tree

‎sagemaker-core/src/sagemaker/core/utils/utils.py‎

Lines changed: 90 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import os
1717
import re
1818
import subprocess
19+
import threading
20+
from collections import OrderedDict
1921

2022
from boto3.session import Session
2123
from botocore.config import Config
@@ -333,15 +335,92 @@ def __call__(cls, *args, **kwargs):
333335
return cls._instances[cls]
334336

335337

336-
class SageMakerClient(metaclass=SingletonMeta):
338+
class _ClientCacheMeta(type):
337339
"""
338-
A singleton class for creating a SageMaker client.
340+
Metaclass that caches instances per constructor arguments.
341+
342+
A call with no arguments returns the process-wide default instance, creating it
343+
from the default credential chain if none exists yet. A call with an explicit
344+
``session``, ``region_name`` or ``config`` returns an instance built from exactly
345+
those arguments, cached so repeated calls with the same objects are cheap. The
346+
first instance ever created also becomes the default, so code that configures
347+
``SageMakerClient(session=...)`` once up front and then relies on bare
348+
``SageMakerClient()`` calls keeps working.
349+
350+
The keyed entries form a small LRU: once ``_MAX_KEYED_ENTRIES`` distinct
351+
argument sets have been seen, the least recently used one is dropped, so a
352+
long-lived process that mints a fresh session per unit of work does not retain
353+
every session and its four boto3 clients forever. The default entry is never
354+
evicted. Construction is serialized by a lock so concurrent first callers for
355+
the same key share one instance.
356+
357+
This replaces a plain singleton, which returned whichever instance was created
358+
first and silently ignored the session passed on every later call.
359+
"""
360+
361+
_DEFAULT_KEY = "default"
362+
_MAX_KEYED_ENTRIES = 32
363+
_instances: Dict[type, "OrderedDict[Any, Any]"] = {}
364+
_lock = threading.RLock()
365+
366+
def __call__(cls, *args, **kwargs):
367+
"""Return a cached instance for these arguments, creating it if needed."""
368+
key = cls._cache_key(*args, **kwargs)
369+
with cls._lock:
370+
cache = cls._instances.setdefault(cls, OrderedDict())
371+
instance = cache.get(key)
372+
if instance is not None:
373+
if key != cls._DEFAULT_KEY:
374+
cache.move_to_end(key)
375+
return instance
376+
instance = super().__call__(*args, **kwargs)
377+
cache[key] = instance
378+
# The first client created in the process becomes the default used by
379+
# argument-less calls, matching the previous singleton behaviour.
380+
cache.setdefault(cls._DEFAULT_KEY, instance)
381+
_ClientCacheMeta._evict(cache, cls._DEFAULT_KEY, cls._MAX_KEYED_ENTRIES)
382+
return instance
383+
384+
@staticmethod
385+
def _evict(cache, default_key, max_keyed_entries):
386+
"""Drop least recently used keyed entries beyond the cap; keep the default."""
387+
keyed = [k for k in cache if k != default_key]
388+
for key in keyed[: max(0, len(keyed) - max_keyed_entries)]:
389+
del cache[key]
390+
391+
392+
class SageMakerClient(metaclass=_ClientCacheMeta):
393+
"""
394+
Cached factory for SageMaker boto3 clients.
395+
396+
Clients are cached per (session, region_name, config). Passing an explicit boto3
397+
``session`` always yields clients signed with that session's credentials, even if
398+
a client for a different session was created earlier in the process. Calling
399+
``SageMakerClient()`` with no arguments returns the process default.
339400
"""
340401

402+
@classmethod
403+
def _cache_key(cls, session: Session = None, region_name: str = None, config: Config = None):
404+
"""Build the cache key for a constructor call.
405+
406+
Sessions and configs are keyed by identity: two distinct boto3 sessions must
407+
never share a client even if they look alike, because credentials live on the
408+
session object. The cached instance keeps references to both, so the ids
409+
cannot be recycled while the entry is alive.
410+
"""
411+
if session is None and region_name is None and config is None:
412+
return cls._DEFAULT_KEY
413+
return (
414+
id(session) if session is not None else None,
415+
region_name,
416+
id(config) if config is not None else None,
417+
)
418+
341419
@classmethod
342420
def reset(cls):
343-
"""Reset the singleton instance."""
344-
SingletonMeta._instances.pop(cls, None)
421+
"""Drop every cached instance, including the default."""
422+
with _ClientCacheMeta._lock:
423+
_ClientCacheMeta._instances.pop(cls, None)
345424

346425
def __init__(
347426
self,
@@ -365,13 +444,16 @@ def __init__(
365444
logger.debug("No config provided. Using default config.")
366445
config = Config(retries={"max_attempts": 10, "mode": "standard"})
367446

368-
self.config = Config(user_agent_extra=get_user_agent_extra_suffix())
447+
# Keep the caller's config object alive: the cache key uses its identity.
448+
self._base_config = config
449+
user_agent_extra = get_user_agent_extra_suffix()
450+
if config.user_agent_extra:
451+
user_agent_extra = f"{config.user_agent_extra} {user_agent_extra}"
452+
self.config = config.merge(Config(user_agent_extra=user_agent_extra))
369453
self.session = session
370454
self.region_name = region_name
371455

372-
self.sagemaker_client = session.client(
373-
"sagemaker", region_name, config=self.config
374-
)
456+
self.sagemaker_client = session.client("sagemaker", region_name, config=self.config)
375457
self.sagemaker_runtime_client = session.client(
376458
"sagemaker-runtime", region_name, config=self.config
377459
)
Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
"""Tests for SageMakerClient caching and session handling.
14+
15+
Regression coverage for aws/sagemaker-python-sdk#5986 and #6069: a client built
16+
for one boto3 session must never be handed to a caller that passed a different
17+
session, and the ``config`` argument must actually reach the boto3 clients.
18+
"""
19+
20+
from __future__ import absolute_import
21+
22+
import boto3
23+
import pytest
24+
from botocore.config import Config
25+
26+
from sagemaker.core.utils.utils import SageMakerClient
27+
28+
REGION_A = "us-west-2"
29+
REGION_B = "us-east-1"
30+
31+
32+
def _session(access_key, region):
33+
"""Offline boto3 session carrying distinguishable static credentials."""
34+
return boto3.Session(
35+
aws_access_key_id=access_key,
36+
aws_secret_access_key="secret-" + access_key,
37+
region_name=region,
38+
)
39+
40+
41+
def _access_key(boto_client):
42+
return boto_client._request_signer._credentials.access_key
43+
44+
45+
@pytest.fixture(autouse=True)
46+
def _isolated_cache():
47+
SageMakerClient.reset()
48+
yield
49+
SageMakerClient.reset()
50+
51+
52+
def test_explicit_session_is_honored_after_another_client_exists():
53+
"""The second session must not receive the first session's client (#5986, #6069)."""
54+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
55+
session_b = _session("AKIASECONDSESSION000", REGION_B)
56+
57+
client_a = SageMakerClient(session=session_a)
58+
client_b = SageMakerClient(session=session_b)
59+
60+
assert client_a is not client_b
61+
assert client_b.session is session_b
62+
assert client_b.region_name == REGION_B
63+
for service in (
64+
"sagemaker",
65+
"sagemaker-runtime",
66+
"sagemaker-featurestore-runtime",
67+
"sagemaker-metrics",
68+
):
69+
assert _access_key(client_b.get_client(service)) == "AKIASECONDSESSION000"
70+
assert _access_key(client_a.get_client(service)) == "AKIAFIRSTSESSION0000"
71+
72+
73+
def test_same_session_returns_cached_instance():
74+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
75+
76+
assert SageMakerClient(session=session_a) is SageMakerClient(session=session_a)
77+
assert SageMakerClient(session=session_a, region_name=REGION_A) is SageMakerClient(
78+
session=session_a, region_name=REGION_A
79+
)
80+
81+
82+
def test_same_session_different_region_gets_distinct_client():
83+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
84+
85+
client_a = SageMakerClient(session=session_a, region_name=REGION_A)
86+
client_b = SageMakerClient(session=session_a, region_name=REGION_B)
87+
88+
assert client_a is not client_b
89+
assert client_b.sagemaker_client.meta.region_name == REGION_B
90+
91+
92+
def test_bare_call_returns_first_created_instance_as_default():
93+
"""Configure-once pattern: SageMakerClient(session=...) then bare SageMakerClient()."""
94+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
95+
96+
configured = SageMakerClient(session=session_a)
97+
98+
assert SageMakerClient() is configured
99+
assert SageMakerClient() is configured
100+
101+
102+
def test_bare_call_default_is_not_replaced_by_later_explicit_session():
103+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
104+
session_b = _session("AKIASECONDSESSION000", REGION_B)
105+
106+
default = SageMakerClient(session=session_a)
107+
SageMakerClient(session=session_b)
108+
109+
assert SageMakerClient() is default
110+
111+
112+
def test_bare_call_creates_default_from_default_chain(monkeypatch):
113+
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "AKIAENVIRONMENT00000")
114+
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "secret")
115+
monkeypatch.setenv("AWS_DEFAULT_REGION", REGION_A)
116+
117+
default = SageMakerClient()
118+
119+
assert SageMakerClient() is default
120+
assert default.region_name == REGION_A
121+
assert _access_key(default.sagemaker_client) == "AKIAENVIRONMENT00000"
122+
123+
124+
def test_config_argument_is_applied_and_user_agent_suffix_appended():
125+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
126+
config = Config(connect_timeout=3, read_timeout=7, user_agent_extra="mytool/1.0")
127+
128+
client = SageMakerClient(session=session_a, config=config)
129+
130+
meta_config = client.sagemaker_client.meta.config
131+
assert meta_config.connect_timeout == 3
132+
assert meta_config.read_timeout == 7
133+
assert meta_config.user_agent_extra.startswith("mytool/1.0 ")
134+
assert "sagemaker" in meta_config.user_agent_extra.lower()
135+
136+
137+
def test_default_config_keeps_retry_settings():
138+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
139+
140+
client = SageMakerClient(session=session_a)
141+
142+
# botocore normalizes {"max_attempts": N} to {"total_max_attempts": N + 1}.
143+
retries = client.sagemaker_client.meta.config.retries
144+
assert retries["total_max_attempts"] == 11
145+
assert retries["mode"] == "standard"
146+
147+
148+
def test_distinct_config_objects_get_distinct_clients():
149+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
150+
config_1 = Config(connect_timeout=1)
151+
config_2 = Config(connect_timeout=2)
152+
153+
client_1 = SageMakerClient(session=session_a, config=config_1)
154+
client_2 = SageMakerClient(session=session_a, config=config_2)
155+
156+
assert client_1 is not client_2
157+
assert client_1.sagemaker_client.meta.config.connect_timeout == 1
158+
assert client_2.sagemaker_client.meta.config.connect_timeout == 2
159+
160+
161+
def test_reset_clears_default_and_keyed_entries():
162+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
163+
client_a = SageMakerClient(session=session_a)
164+
default = SageMakerClient()
165+
166+
SageMakerClient.reset()
167+
168+
assert SageMakerClient(session=session_a) is not client_a
169+
assert SageMakerClient() is not default
170+
171+
172+
def _cached_entries():
173+
from sagemaker.core.utils.utils import _ClientCacheMeta
174+
175+
return _ClientCacheMeta._instances.get(SageMakerClient, {})
176+
177+
178+
def test_keyed_cache_is_bounded_and_default_survives_eviction():
179+
from sagemaker.core.utils.utils import _ClientCacheMeta
180+
181+
cap = _ClientCacheMeta._MAX_KEYED_ENTRIES
182+
sessions = [_session(f"AKIA{i:016d}", REGION_A) for i in range(cap + 5)]
183+
184+
default = SageMakerClient(session=sessions[0])
185+
clients = [SageMakerClient(session=s) for s in sessions]
186+
187+
entries = _cached_entries()
188+
keyed = [k for k in entries if k != _ClientCacheMeta._DEFAULT_KEY]
189+
assert len(keyed) == cap
190+
# The oldest keyed entries were dropped, the newest kept.
191+
assert SageMakerClient(session=sessions[-1]) is clients[-1]
192+
assert SageMakerClient(session=sessions[0]) is not clients[0]
193+
# The process default is pinned regardless of eviction.
194+
assert SageMakerClient() is default
195+
196+
197+
def test_recently_used_entry_is_kept_over_stale_ones():
198+
from sagemaker.core.utils.utils import _ClientCacheMeta
199+
200+
cap = _ClientCacheMeta._MAX_KEYED_ENTRIES
201+
sessions = [_session(f"AKIA{i:016d}", REGION_A) for i in range(cap)]
202+
clients = [SageMakerClient(session=s) for s in sessions]
203+
204+
# Touch the oldest entry so it becomes most recently used, then overflow by one.
205+
assert SageMakerClient(session=sessions[0]) is clients[0]
206+
SageMakerClient(session=_session("AKIAOVERFLOW00000000", REGION_A))
207+
208+
assert SageMakerClient(session=sessions[0]) is clients[0]
209+
assert SageMakerClient(session=sessions[1]) is not clients[1]
210+
211+
212+
def test_concurrent_first_calls_share_one_instance():
213+
import threading
214+
215+
session_a = _session("AKIAFIRSTSESSION0000", REGION_A)
216+
results = []
217+
start = threading.Barrier(8)
218+
219+
def build():
220+
start.wait()
221+
results.append(SageMakerClient(session=session_a))
222+
223+
threads = [threading.Thread(target=build) for _ in range(8)]
224+
for t in threads:
225+
t.start()
226+
for t in threads:
227+
t.join()
228+
229+
assert len(results) == 8
230+
assert all(r is results[0] for r in results)

0 commit comments

Comments
 (0)