1616import os
1717import platform
1818import sys
19+ import threading
1920from time import perf_counter
2021from typing import List
2122import functools
6667)
6768_telemetry_msg_shown = False
6869
70+ # Seconds to wait for the telemetry endpoint before giving up on an event.
71+ TELEMETRY_REQUEST_TIMEOUT = 2
72+
73+ # Telemetry must never sit in the caller's critical path, so every event is sent
74+ # from a daemon thread. A slow or unreachable telemetry endpoint (for example from
75+ # inside a VPC with no route to it) can no longer stall the SDK call the customer
76+ # actually made, and because the threads are daemons a pending send cannot delay
77+ # interpreter shutdown either. No event is ever dropped: one thread is started per
78+ # event.
79+ #
80+ # Special case worth calling out: Feature Store ingestion is decorated at more than
81+ # one level (``ingest_dataframe`` and ``IngestionManagerPandas.run``), so a single
82+ # user call can emit several events. Sending them serially on the caller's thread is
83+ # what turned an ingest that the service finished in under a second into a
84+ # multi-minute wait, which is why the send is moved off that thread here rather than
85+ # only having its timeout tightened.
86+
6987FEATURE_TO_CODE = {
7088 str (Feature .SDK_DEFAULTS ): 11 ,
7189 str (Feature .LOCAL_MODE ): 12 ,
@@ -422,6 +440,41 @@ def _send_telemetry_request(
422440 failure_reason : str = None ,
423441 failure_type : str = None ,
424442 extra_info : str = None ,
443+ ) -> threading .Thread :
444+ """Schedule a telemetry event to be sent on a background daemon thread.
445+
446+ Every event is still sent; this only moves the send off the caller's thread so
447+ that telemetry never adds latency to the SDK call that triggered it.
448+
449+ Returns:
450+ threading.Thread: The thread doing the send. Callers generally ignore this;
451+ tests can join on it.
452+ """
453+
454+ def _run ():
455+ try :
456+ _send_telemetry_request_sync (
457+ status , feature_list , session , failure_reason , failure_type , extra_info
458+ )
459+ except Exception : # pylint: disable=W0703
460+ # Nothing can be raised out of a fire-and-forget thread: there is no
461+ # caller to catch it, and even the logging call inside
462+ # _send_telemetry_request_sync can fail once the interpreter starts
463+ # tearing down. Telemetry is best-effort, so drop the event silently.
464+ pass
465+
466+ thread = threading .Thread (target = _run , name = "sagemaker-telemetry" , daemon = True )
467+ thread .start ()
468+ return thread
469+
470+
471+ def _send_telemetry_request_sync (
472+ status : int ,
473+ feature_list : List [int ],
474+ session : Session ,
475+ failure_reason : str = None ,
476+ failure_type : str = None ,
477+ extra_info : str = None ,
425478) -> None :
426479 """Make GET request to an empty object in S3 bucket"""
427480 try :
@@ -449,7 +502,7 @@ def _send_telemetry_request(
449502 )
450503 # Send the telemetry request
451504 logger .debug ("Sending telemetry request to [%s]" , url )
452- _requests_helper (url , 2 )
505+ _requests_helper (url , TELEMETRY_REQUEST_TIMEOUT )
453506 logger .debug ("SageMaker Python SDK telemetry successfully emitted." )
454507 except Exception : # pylint: disable=W0703
455508 logger .debug ("SageMaker Python SDK telemetry not emitted!" )
@@ -482,12 +535,17 @@ def _construct_url(
482535
483536
484537def _requests_helper (url , timeout ):
485- """Make a GET request to the given URL"""
538+ """Make a GET request to the given URL
539+
540+ ``timeout`` must be passed by keyword. ``requests.get`` takes ``params`` as
541+ its second positional argument, so passing it positionally would append the
542+ value to the query string and leave the request with no timeout at all.
543+ """
486544 response = None
487545 try :
488- response = requests .get (url , timeout )
546+ response = requests .get (url , timeout = timeout )
489547 except requests .exceptions .RequestException as e :
490- logger .exception ("Request exception: %s" , str (e ))
548+ logger .debug ("Request exception: %s" , str (e ))
491549 return response
492550
493551
@@ -511,9 +569,24 @@ def _get_region_or_default(session):
511569
512570
513571def _get_default_sagemaker_session ():
514- """Return the default sagemaker session"""
572+ """Return the default sagemaker session
573+
574+ The region is resolved by boto3 from the caller's own environment
575+ (``AWS_REGION``, ``AWS_DEFAULT_REGION``, or the active profile in
576+ ``~/.aws/config``). ``DEFAULT_AWS_REGION`` is only used as a last resort,
577+ because ``Session`` requires a region. Hardcoding the default meant that
578+ callers with no session of their own (module-level functions such as
579+ ``ingest_dataframe``) had their telemetry pointed at a region they may have
580+ no network route to.
581+ """
515582
516- boto_session = boto3 .Session (region_name = DEFAULT_AWS_REGION )
583+ boto_session = boto3 .Session ()
584+ if not boto_session .region_name :
585+ logger .debug (
586+ "No region resolved from the local AWS configuration. Falling back to %s." ,
587+ DEFAULT_AWS_REGION ,
588+ )
589+ boto_session = boto3 .Session (region_name = DEFAULT_AWS_REGION )
517590 sagemaker_session = Session (boto_session = boto_session )
518591
519592 return sagemaker_session
0 commit comments