Skip to content

Commit 2c9fd13

Browse files
rsareddy0329Roja Reddy Sareddy
andauthored
Feat: finetuningoptions validation telemetry (#6286)
* feat(train): emit telemetry on FineTuningOptions validation failures Instrument FineTuningOptions.__setattr__ so client-side hyperparameter validation failures are captured in the SDK dev-experience logs, enabling detection and measurement of this class of failure: - setting an option name that is not a valid/overridable hyperparameter (raises AttributeError), and - setting a value outside its spec (type/range/enum; raises ValueError). Emission is failure-only: a single MODEL_CUSTOMIZATION FAILURE event is sent on the failure path, and nothing is emitted on successful sets, internal attribute sets, or construction, so no telemetry noise is added to the happy path. The emit is best-effort and never masks or replaces the user-facing validation error. Add unit tests covering both failure paths, the no-emit happy paths, and that a telemetry backend error still lets the validation error propagate. * refactor(core): add reusable _emit_failure_telemetry helper Move the failure-only telemetry emit out of FineTuningOptions and into the core telemetry module as a reusable _emit_failure_telemetry(feature, func_name, exc, sagemaker_session=None) helper. Unlike the @_telemetry_emitter decorator, which emits on success and failure for every call, this emits a single FAILURE event only when a caller explicitly hits a failure -- so it adds no happy-path telemetry or per-call overhead. It is best-effort and never raises. FineTuningOptions.__setattr__ now calls the shared helper on its two validation failure paths (invalid option name, out-of-spec value) instead of carrying its own emit plumbing, which shrinks the common.py change to a single import and two call sites. Add unit tests for the helper in the core telemetry suite and update the common.py tests to assert the helper is invoked on failures only. * fix(core): honor telemetry opt-out in _emit_failure_telemetry Gate _emit_failure_telemetry on the TelemetryOptOut config flag (via resolve_value_from_config), matching the @_telemetry_emitter decorator, so a user who has opted out of telemetry does not have validation-failure events emitted. Also include the standard env/platform dimensions (x-env, x-sys, x-platform) in the payload for parity with decorator-emitted events, so the two can be sliced consistently. Add a unit test asserting no event is sent when opt-out is set. --------- Co-authored-by: Roja Reddy Sareddy <rsareddy@amazon.com>
1 parent 8bb160f commit 2c9fd13

4 files changed

Lines changed: 217 additions & 3 deletions

File tree

‎sagemaker-core/src/sagemaker/core/telemetry/telemetry_logging.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -471,6 +471,65 @@ def _run():
471471
return thread
472472

473473

474+
def _emit_failure_telemetry(
475+
feature: str,
476+
func_name: str,
477+
exc: Exception,
478+
sagemaker_session: Session = None,
479+
) -> None:
480+
"""Emit a single FAILURE telemetry event for a client-side failure.
481+
482+
Unlike the ``@_telemetry_emitter`` decorator -- which wraps a call and emits on
483+
both success and failure for every invocation -- this helper emits only when a
484+
caller has explicitly hit a failure it wants recorded. Use it to capture a class
485+
of client-side failure (e.g. invalid user input) without adding any happy-path
486+
telemetry or per-call overhead to the surrounding code.
487+
488+
Best-effort: it resolves a session (falling back to the default), honors the
489+
telemetry opt-out configuration, and swallows any error while emitting, so it can
490+
never mask or replace the caller's own exception.
491+
492+
Args:
493+
feature: The Feature enum value to attribute this event to.
494+
func_name: Human-readable name of the failing operation, for tracking.
495+
exc: The exception representing the failure (used for reason/type/category).
496+
sagemaker_session: Optional session; the default session is used if omitted.
497+
"""
498+
try:
499+
session = sagemaker_session or _get_default_sagemaker_session()
500+
if not session:
501+
return
502+
# Honor the same telemetry opt-out contract as @_telemetry_emitter: a user
503+
# who has opted out must not have these events emitted.
504+
if resolve_value_from_config(
505+
direct_input=None,
506+
config_path=TELEMETRY_OPT_OUT_PATH,
507+
default_value=False,
508+
sagemaker_session=session,
509+
):
510+
return
511+
# Mirror the decorator's platform/env dimensions so these events can be
512+
# sliced consistently alongside decorator-emitted ones.
513+
extra = (
514+
f"{func_name}"
515+
f"&x-sdkVersion={SDK_VERSION}"
516+
f"&x-env={PYTHON_VERSION}"
517+
f"&x-sys={OS_NAME_VERSION}"
518+
f"&x-platform={process_studio_metadata_file()}"
519+
f"&x-errorCategory={_classify_error(exc)}"
520+
)
521+
_send_telemetry_request(
522+
STATUS_TO_CODE[str(Status.FAILURE)],
523+
[FEATURE_TO_CODE[str(feature)]],
524+
session,
525+
str(exc),
526+
exc.__class__.__name__,
527+
extra,
528+
)
529+
except Exception: # pragma: no cover - telemetry must never break the caller
530+
pass
531+
532+
474533
def _send_telemetry_request_sync(
475534
status: int,
476535
feature_list: List[int],

‎sagemaker-core/tests/unit/telemetry/test_telemetry_logging.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,19 @@
2525
from sagemaker.core.telemetry.telemetry_logging import (
2626
_send_telemetry_request,
2727
_send_telemetry_request_sync,
28+
_emit_failure_telemetry,
2829
_telemetry_emitter,
2930
_construct_url,
3031
_get_accountId,
3132
_requests_helper,
3233
_get_region_or_default,
3334
_get_default_sagemaker_session,
35+
STATUS_TO_CODE,
3436
OS_NAME_VERSION,
3537
PYTHON_VERSION,
3638
TELEMETRY_REQUEST_TIMEOUT,
3739
)
40+
from sagemaker.core.telemetry.constants import Status
3841
from sagemaker.core.user_agent import SDK_VERSION, process_studio_metadata_file
3942

4043
# Try to import sagemaker-serve exceptions, skip tests if not available
@@ -885,3 +888,69 @@ def test_falls_back_to_default_region_when_none_resolved(self, mock_boto_session
885888
mock_boto_session.call_args_list[-1],
886889
unittest.mock.call(region_name=DEFAULT_AWS_REGION),
887890
)
891+
892+
893+
class TestEmitFailureTelemetry(unittest.TestCase):
894+
"""Tests for the failure-only _emit_failure_telemetry helper."""
895+
896+
@patch("sagemaker.core.telemetry.telemetry_logging.resolve_value_from_config", return_value=False)
897+
@patch("sagemaker.core.telemetry.telemetry_logging._get_default_sagemaker_session")
898+
@patch("sagemaker.core.telemetry.telemetry_logging._send_telemetry_request")
899+
def test_emits_failure_event(self, mock_send, mock_default_session, mock_optout):
900+
mock_default_session.return_value = Mock()
901+
exc = ValueError("bad value")
902+
903+
_emit_failure_telemetry(Feature.MODEL_CUSTOMIZATION, "MyClass.method", exc)
904+
905+
mock_send.assert_called_once()
906+
args = mock_send.call_args.args
907+
assert args[0] == STATUS_TO_CODE[str(Status.FAILURE)] # status code
908+
assert args[3] == "bad value" # failure_reason
909+
assert args[4] == "ValueError" # failure_type
910+
assert "MyClass.method" in args[5] # extra_info carries func_name
911+
912+
@patch("sagemaker.core.telemetry.telemetry_logging.resolve_value_from_config", return_value=True)
913+
@patch("sagemaker.core.telemetry.telemetry_logging._get_default_sagemaker_session")
914+
@patch("sagemaker.core.telemetry.telemetry_logging._send_telemetry_request")
915+
def test_opt_out_suppresses_emit(self, mock_send, mock_default_session, mock_optout):
916+
# A user who has opted out of telemetry must not have these events emitted.
917+
mock_default_session.return_value = Mock()
918+
919+
_emit_failure_telemetry(Feature.MODEL_CUSTOMIZATION, "MyClass.method", ValueError("x"))
920+
921+
mock_send.assert_not_called()
922+
923+
@patch("sagemaker.core.telemetry.telemetry_logging.resolve_value_from_config", return_value=False)
924+
@patch("sagemaker.core.telemetry.telemetry_logging._get_default_sagemaker_session")
925+
@patch("sagemaker.core.telemetry.telemetry_logging._send_telemetry_request")
926+
def test_no_session_does_not_emit(self, mock_send, mock_default_session, mock_optout):
927+
mock_default_session.return_value = None
928+
929+
_emit_failure_telemetry(Feature.MODEL_CUSTOMIZATION, "MyClass.method", ValueError("x"))
930+
931+
mock_send.assert_not_called()
932+
933+
@patch("sagemaker.core.telemetry.telemetry_logging.resolve_value_from_config", return_value=False)
934+
@patch("sagemaker.core.telemetry.telemetry_logging._get_default_sagemaker_session")
935+
@patch("sagemaker.core.telemetry.telemetry_logging._send_telemetry_request")
936+
def test_uses_provided_session_without_default_lookup(
937+
self, mock_send, mock_default_session, mock_optout
938+
):
939+
_emit_failure_telemetry(
940+
Feature.MODEL_CUSTOMIZATION, "MyClass.method", ValueError("x"),
941+
sagemaker_session=Mock(),
942+
)
943+
944+
mock_default_session.assert_not_called()
945+
mock_send.assert_called_once()
946+
947+
@patch("sagemaker.core.telemetry.telemetry_logging.resolve_value_from_config", return_value=False)
948+
@patch("sagemaker.core.telemetry.telemetry_logging._get_default_sagemaker_session")
949+
@patch("sagemaker.core.telemetry.telemetry_logging._send_telemetry_request")
950+
def test_backend_error_is_swallowed(self, mock_send, mock_default_session, mock_optout):
951+
# Telemetry is best-effort: an error while emitting must never propagate.
952+
mock_default_session.return_value = Mock()
953+
mock_send.side_effect = RuntimeError("telemetry backend down")
954+
955+
# Should not raise.
956+
_emit_failure_telemetry(Feature.MODEL_CUSTOMIZATION, "MyClass.method", ValueError("x"))

‎sagemaker-train/src/sagemaker/train/common.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
from typing import Dict, Any
22
from enum import Enum
3-
from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter
3+
from sagemaker.core.telemetry.telemetry_logging import (
4+
_telemetry_emitter,
5+
_emit_failure_telemetry,
6+
)
47
from sagemaker.core.telemetry.constants import Feature
58

69
JOB_TYPE = "FineTuning"
@@ -93,11 +96,24 @@ def __setattr__(self, name: str, value: Any):
9396
if getattr(self, '_initialized', False):
9497
spec = self._specs[name]
9598
if isinstance(spec, dict):
96-
self._validate_value(name, value, spec)
99+
try:
100+
self._validate_value(name, value, spec)
101+
except Exception as exc:
102+
_emit_failure_telemetry(
103+
Feature.MODEL_CUSTOMIZATION, "FineTuningOptions.__setattr__", exc
104+
)
105+
raise
97106
self._user_set.add(name)
98107
super().__setattr__(name, value)
99108
elif hasattr(self, '_specs'):
100-
raise AttributeError(f"'{name}' is not a valid fine-tuning option. Valid options: {list(self._specs.keys())}")
109+
exc = AttributeError(
110+
f"'{name}' is not a valid fine-tuning option. "
111+
f"Valid options: {list(self._specs.keys())}"
112+
)
113+
_emit_failure_telemetry(
114+
Feature.MODEL_CUSTOMIZATION, "FineTuningOptions.__setattr__", exc
115+
)
116+
raise exc
101117
else:
102118
super().__setattr__(name, value)
103119

‎sagemaker-train/tests/unit/train/test_common.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
from unittest.mock import patch
2+
3+
import pytest
4+
15
from sagemaker.train.common import FineTuningOptions
26

37

@@ -129,3 +133,69 @@ def test_framework_agnostic_gated_on_sequence_length_metadata_only(self):
129133
)
130134
object.__setattr__(opts, "dataset_max_len", 999999)
131135
opts.validate_length_constraints() # unknown ceiling -> no raise
136+
137+
138+
class TestFineTuningOptionsValidationTelemetry:
139+
"""Failure-only telemetry emitted from FineTuningOptions.__setattr__.
140+
141+
A single FAILURE event (via the core _emit_failure_telemetry helper) is emitted
142+
when a caller sets an invalid option name or an out-of-spec value, and NOTHING is
143+
emitted on the happy path (valid sets, internal sets, or construction).
144+
"""
145+
146+
_SPECS = {
147+
"learning_rate": {"default": 1e-4, "type": "float", "min": 1e-7, "max": 1.0},
148+
"num_epochs": {"default": 3, "type": "integer", "min": 1, "max": 100},
149+
}
150+
151+
@patch("sagemaker.train.common._emit_failure_telemetry")
152+
def test_invalid_option_name_emits_failure(self, mock_emit):
153+
options = FineTuningOptions(self._SPECS)
154+
with pytest.raises(AttributeError):
155+
options.not_a_real_param = 5
156+
157+
mock_emit.assert_called_once()
158+
args = mock_emit.call_args[0]
159+
assert args[1] == "FineTuningOptions.__setattr__" # func_name
160+
assert isinstance(args[2], AttributeError) # exc
161+
162+
@patch("sagemaker.train.common._emit_failure_telemetry")
163+
def test_out_of_range_value_emits_failure(self, mock_emit):
164+
options = FineTuningOptions(self._SPECS)
165+
with pytest.raises(ValueError):
166+
options.learning_rate = 999.0 # exceeds max=1.0
167+
168+
mock_emit.assert_called_once()
169+
assert isinstance(mock_emit.call_args[0][2], ValueError)
170+
171+
@patch("sagemaker.train.common._emit_failure_telemetry")
172+
def test_wrong_type_value_emits_failure(self, mock_emit):
173+
options = FineTuningOptions(self._SPECS)
174+
with pytest.raises(ValueError):
175+
options.num_epochs = "three" # not an integer
176+
177+
mock_emit.assert_called_once()
178+
assert isinstance(mock_emit.call_args[0][2], ValueError)
179+
180+
@patch("sagemaker.train.common._emit_failure_telemetry")
181+
def test_valid_set_does_not_emit(self, mock_emit):
182+
options = FineTuningOptions(self._SPECS)
183+
options.learning_rate = 1e-3 # within range -> no telemetry
184+
assert options.learning_rate == 1e-3
185+
mock_emit.assert_not_called()
186+
187+
@patch("sagemaker.train.common._emit_failure_telemetry")
188+
def test_construction_does_not_emit(self, mock_emit):
189+
# __init__ sets defaults via super().__setattr__ and internal _-prefixed attrs,
190+
# none of which should emit telemetry.
191+
FineTuningOptions(self._SPECS)
192+
mock_emit.assert_not_called()
193+
194+
@patch("sagemaker.train.common._emit_failure_telemetry")
195+
def test_emits_model_customization_feature(self, mock_emit):
196+
from sagemaker.core.telemetry.constants import Feature
197+
198+
options = FineTuningOptions(self._SPECS)
199+
with pytest.raises(AttributeError):
200+
options.bogus = 1
201+
assert mock_emit.call_args[0][0] == Feature.MODEL_CUSTOMIZATION

0 commit comments

Comments
 (0)