Skip to content

Commit 6593984

Browse files
authored
Merge branch 'master' into fix/jumpstart-private-hub-modeltrainer
2 parents a63d8b6 + a248126 commit 6593984

9 files changed

Lines changed: 582 additions & 25 deletions

File tree

‎sagemaker-core/src/sagemaker/core/telemetry/telemetry_logging.py‎

Lines changed: 79 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import os
1717
import platform
1818
import sys
19+
import threading
1920
from time import perf_counter
2021
from typing import List
2122
import functools
@@ -66,6 +67,23 @@
6667
)
6768
_telemetry_msg_shown = False
6869

70+
# Seconds to wait for the telemetry endpoint before giving up on an event.
71+
TELEMETRY_REQUEST_TIMEOUT = 2
72+
73+
# Telemetry must never sit in the caller's critical path, so every event is sent
74+
# from a daemon thread. A slow or unreachable telemetry endpoint (for example from
75+
# inside a VPC with no route to it) can no longer stall the SDK call the customer
76+
# actually made, and because the threads are daemons a pending send cannot delay
77+
# interpreter shutdown either. No event is ever dropped: one thread is started per
78+
# event.
79+
#
80+
# Special case worth calling out: Feature Store ingestion is decorated at more than
81+
# one level (``ingest_dataframe`` and ``IngestionManagerPandas.run``), so a single
82+
# user call can emit several events. Sending them serially on the caller's thread is
83+
# what turned an ingest that the service finished in under a second into a
84+
# multi-minute wait, which is why the send is moved off that thread here rather than
85+
# only having its timeout tightened.
86+
6987
FEATURE_TO_CODE = {
7088
str(Feature.SDK_DEFAULTS): 11,
7189
str(Feature.LOCAL_MODE): 12,
@@ -422,6 +440,41 @@ def _send_telemetry_request(
422440
failure_reason: str = None,
423441
failure_type: str = None,
424442
extra_info: str = None,
443+
) -> threading.Thread:
444+
"""Schedule a telemetry event to be sent on a background daemon thread.
445+
446+
Every event is still sent; this only moves the send off the caller's thread so
447+
that telemetry never adds latency to the SDK call that triggered it.
448+
449+
Returns:
450+
threading.Thread: The thread doing the send. Callers generally ignore this;
451+
tests can join on it.
452+
"""
453+
454+
def _run():
455+
try:
456+
_send_telemetry_request_sync(
457+
status, feature_list, session, failure_reason, failure_type, extra_info
458+
)
459+
except Exception: # pylint: disable=W0703
460+
# Nothing can be raised out of a fire-and-forget thread: there is no
461+
# caller to catch it, and even the logging call inside
462+
# _send_telemetry_request_sync can fail once the interpreter starts
463+
# tearing down. Telemetry is best-effort, so drop the event silently.
464+
pass
465+
466+
thread = threading.Thread(target=_run, name="sagemaker-telemetry", daemon=True)
467+
thread.start()
468+
return thread
469+
470+
471+
def _send_telemetry_request_sync(
472+
status: int,
473+
feature_list: List[int],
474+
session: Session,
475+
failure_reason: str = None,
476+
failure_type: str = None,
477+
extra_info: str = None,
425478
) -> None:
426479
"""Make GET request to an empty object in S3 bucket"""
427480
try:
@@ -449,7 +502,7 @@ def _send_telemetry_request(
449502
)
450503
# Send the telemetry request
451504
logger.debug("Sending telemetry request to [%s]", url)
452-
_requests_helper(url, 2)
505+
_requests_helper(url, TELEMETRY_REQUEST_TIMEOUT)
453506
logger.debug("SageMaker Python SDK telemetry successfully emitted.")
454507
except Exception: # pylint: disable=W0703
455508
logger.debug("SageMaker Python SDK telemetry not emitted!")
@@ -482,12 +535,17 @@ def _construct_url(
482535

483536

484537
def _requests_helper(url, timeout):
485-
"""Make a GET request to the given URL"""
538+
"""Make a GET request to the given URL
539+
540+
``timeout`` must be passed by keyword. ``requests.get`` takes ``params`` as
541+
its second positional argument, so passing it positionally would append the
542+
value to the query string and leave the request with no timeout at all.
543+
"""
486544
response = None
487545
try:
488-
response = requests.get(url, timeout)
546+
response = requests.get(url, timeout=timeout)
489547
except requests.exceptions.RequestException as e:
490-
logger.exception("Request exception: %s", str(e))
548+
logger.debug("Request exception: %s", str(e))
491549
return response
492550

493551

@@ -511,9 +569,24 @@ def _get_region_or_default(session):
511569

512570

513571
def _get_default_sagemaker_session():
514-
"""Return the default sagemaker session"""
572+
"""Return the default sagemaker session
573+
574+
The region is resolved by boto3 from the caller's own environment
575+
(``AWS_REGION``, ``AWS_DEFAULT_REGION``, or the active profile in
576+
``~/.aws/config``). ``DEFAULT_AWS_REGION`` is only used as a last resort,
577+
because ``Session`` requires a region. Hardcoding the default meant that
578+
callers with no session of their own (module-level functions such as
579+
``ingest_dataframe``) had their telemetry pointed at a region they may have
580+
no network route to.
581+
"""
515582

516-
boto_session = boto3.Session(region_name=DEFAULT_AWS_REGION)
583+
boto_session = boto3.Session()
584+
if not boto_session.region_name:
585+
logger.debug(
586+
"No region resolved from the local AWS configuration. Falling back to %s.",
587+
DEFAULT_AWS_REGION,
588+
)
589+
boto_session = boto3.Session(region_name=DEFAULT_AWS_REGION)
517590
sagemaker_session = Session(boto_session=boto_session)
518591

519592
return sagemaker_session

0 commit comments

Comments
 (0)