Skip to content

Commit 0d811b2

Browse files
author
Rohan Gujarathi
committed
fix: propagate user-supplied tags to created resources
Tags passed to the trainer and evaluator classes were accepted but never reached the resources they create, so they could not be used for tag-based resource association. - Trainers (SFT, DPO, RLVR, RLAIF, MultiTurnRL): the serverless train() path built its tag list from the JumpStart tags only and discarded self.tags. MultiTurnRLTrainer additionally computed tags and then never passed them to Job.create(). - BaseTrainer._train_serverful_smtj() did not forward tags to ModelTrainer.from_recipe(). - BaseEvaluator had no tags field at all; add one and merge it into the pipeline tags. - MultiTurnRLEvaluator passed only the evaluation discovery tag to CreatePipeline and dropped customer tags. The create APIs are typed List[Tag] and validated by pydantic, which coerces plain key/value dicts and accepts Tag objects, so the merged tag lists are passed through as-is. MultiTurnRLEvaluator is the exception: it calls CreatePipeline through raw boto3 and also renders tags into the pipeline definition, both of which require the capitalized Key/Value form, so its tags are normalized locally.
1 parent d768dce commit 0d811b2

12 files changed

Lines changed: 394 additions & 4 deletions

‎sagemaker-train/src/sagemaker/train/base_trainer.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1209,6 +1209,7 @@ def _yaml_safe_default(value):
12091209
sagemaker_session=sagemaker_session,
12101210
role=role,
12111211
base_job_name=base_job_name,
1212+
tags=self.tags,
12121213
)
12131214

12141215
# Validate data paths exist before submission

‎sagemaker-train/src/sagemaker/train/dpo_trainer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,9 @@ def train(self,
346346
vpc_config = self.networking if self.networking else None
347347
tags = _get_jumpstart_tags(self._model_name, get_sagemaker_hub_name())
348348

349+
# Merge user-provided tags with the JumpStart tags
350+
tags.extend(self.tags or [])
351+
349352
# Build TrainingJob.create() arguments
350353
create_args = {
351354
"training_job_name": current_training_job_name,

‎sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,8 @@ class BaseEvaluator(BaseModel):
138138
3. Model package group name string (will fetch the object and extract ARN)
139139
Required when model is a JumpStart model ID. Optional when model is a ModelPackage
140140
ARN/object (will be inferred automatically).
141+
tags (Optional[List[TagsDict]]): Tags applied to the evaluation pipeline when it is
142+
created, which cascade to the pipeline's step jobs.
141143
"""
142144

143145
region: Optional[str] = None
@@ -153,6 +155,7 @@ class BaseEvaluator(BaseModel):
153155
networking: Optional[VpcConfig] = None
154156
kms_key_id: Optional[str] = None
155157
model_package_group: Optional[Union[str, ModelPackageGroup]] = None
158+
tags: Optional[List[TagsDict]] = None
156159
compute: Optional[Union[Compute, HyperPodCompute]] = None
157160
training_image: Optional[str] = None
158161
recipe: Optional[str] = None
@@ -987,6 +990,9 @@ def _start_execution(
987990
if self._is_jumpstart_model:
988991
from sagemaker.core.jumpstart.utils import add_jumpstart_model_info_tags
989992
tags = add_jumpstart_model_info_tags(tags, self.model, "*")
993+
994+
# Merge user-provided tags
995+
tags.extend(self.tags or [])
990996

991997
execution = EvaluationPipelineExecution.start(
992998
eval_type=eval_type,

‎sagemaker-train/src/sagemaker/train/evaluate/multi_turn_rl_evaluator.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,35 @@
4444
_MAX_STOPPING_CONDITION_SECONDS = 72 * 60 * 60
4545

4646

47+
def _tags_with_capitalized_keys(tags: Optional[List[Dict[str, str]]]) -> List[Dict[str, str]]:
48+
"""Convert tags to the capitalized ``Key``/``Value`` form this evaluator's paths require.
49+
50+
Both the raw boto3 ``CreatePipeline`` call and the ``Tags`` block rendered into the
51+
pipeline definition use the API's capitalized form, whereas the other evaluators hand
52+
tags to the pydantic-validated ``Pipeline.create``, which takes the lowercase form.
53+
Either form is accepted here so that the inherited ``tags`` field behaves the same way
54+
across evaluator subclasses.
55+
56+
Args:
57+
tags: Tags using either ``key``/``value`` or ``Key``/``Value``, or None.
58+
59+
Returns:
60+
The tags in capitalized form; empty when none were supplied. Entries missing a key
61+
or value are skipped, since SageMaker rejects them.
62+
"""
63+
normalized = []
64+
for tag in tags or []:
65+
if isinstance(tag, dict):
66+
key = tag.get("Key", tag.get("key"))
67+
value = tag.get("Value", tag.get("value"))
68+
else:
69+
key = getattr(tag, "key", None)
70+
value = getattr(tag, "value", None)
71+
if key is not None and value is not None:
72+
normalized.append({"Key": key, "Value": value})
73+
return normalized
74+
75+
4776
class MultiTurnRLEvaluator(BaseEvaluator):
4877
"""Evaluate a multi-turn RL agent model against a held-out prompt dataset.
4978
@@ -532,7 +561,7 @@ def _build_job_config_doc(include_mpc: bool, mlflow_run_name: str) -> str:
532561
"vpc_config": bool(networking),
533562
"vpc_security_group_ids": vpc_security_group_ids,
534563
"vpc_subnets": vpc_subnets,
535-
"tags": self.tags,
564+
"tags": _tags_with_capitalized_keys(self.tags) or None,
536565
# Pre-stringified JobConfigDocument for the templates.
537566
"job_config_document_str": job_config_doc_str,
538567
"job_config_document_ft_str": job_config_doc_ft_str,
@@ -699,6 +728,11 @@ def _start_mtrl_execution(self, pipeline_definition, name, role_arn, region):
699728
pipeline_prefix = _get_pipeline_name_prefix(EvalType.MTRL)
700729
pipeline_name = pipeline_prefix
701730

731+
# Customer tags are merged into the pipeline tags. This path uses raw boto3, which
732+
# requires the API's capitalized Key/Value form.
733+
pipeline_tags = [{"Key": _TAG_SAGEMAKER_MODEL_EVALUATION, "Value": "true"}]
734+
pipeline_tags.extend(_tags_with_capitalized_keys(self.tags))
735+
702736
# Search for existing MTRL pipeline
703737
existing_pipeline_name = None
704738
try:
@@ -725,7 +759,7 @@ def _start_mtrl_execution(self, pipeline_definition, name, role_arn, region):
725759
PipelineDisplayName=pipeline_name,
726760
PipelineDescription="MTRL evaluation pipeline",
727761
ClientRequestToken=str(uuid.uuid4()),
728-
Tags=[{"Key": _TAG_SAGEMAKER_MODEL_EVALUATION, "Value": "true"}],
762+
Tags=pipeline_tags,
729763
)
730764
_logger.info(f"Created pipeline: {pipeline_name}")
731765

‎sagemaker-train/src/sagemaker/train/multi_turn_rl_trainer.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,13 +321,17 @@ def train(
321321

322322
tags = _get_jumpstart_tags(self._model_name, get_sagemaker_hub_name())
323323

324+
# Merge user-provided tags with the JumpStart tags
325+
tags.extend(self.tags or [])
326+
324327
try:
325328
job = Job.create(
326329
job_name=current_job_name,
327330
job_category=JOB_CATEGORY,
328331
role_arn=role,
329332
job_config_schema_version=JOB_CONFIG_SCHEMA_VERSION,
330333
job_config_document=job_config_doc,
334+
tags=tags,
331335
session=sagemaker_session.boto_session,
332336
region=sagemaker_session.boto_session.region_name,
333337
)

‎sagemaker-train/src/sagemaker/train/rlaif_trainer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,9 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati
310310
vpc_config = self.networking if self.networking else None
311311
tags = _get_jumpstart_tags(self._model_name, get_sagemaker_hub_name())
312312

313+
# Merge user-provided tags with the JumpStart tags
314+
tags.extend(self.tags or [])
315+
313316
# Build TrainingJob.create() arguments
314317
create_args = {
315318
"training_job_name": current_training_job_name,

‎sagemaker-train/src/sagemaker/train/rlvr_trainer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,9 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None,
520520
vpc_config = self.networking if self.networking else None
521521
tags = _get_jumpstart_tags(self._model_name, get_sagemaker_hub_name())
522522

523+
# Merge user-provided tags with the JumpStart tags
524+
tags.extend(self.tags or [])
525+
523526
# Build TrainingJob.create() arguments
524527
create_args = {
525528
"training_job_name": current_training_job_name,

‎sagemaker-train/src/sagemaker/train/sft_trainer.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,9 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, validati
414414
vpc_config = self.networking if self.networking else None
415415
tags = _get_jumpstart_tags(self._model_name, get_sagemaker_hub_name())
416416

417+
# Merge user-provided tags with the JumpStart tags
418+
tags.extend(self.tags or [])
419+
417420
# Build TrainingJob.create() arguments
418421
create_args = {
419422
"training_job_name": current_training_job_name,

‎sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from __future__ import absolute_import
1515

1616
import pytest
17-
from unittest.mock import patch, MagicMock, Mock
17+
from unittest.mock import patch, MagicMock, Mock, PropertyMock
1818
from pydantic import ValidationError
1919

2020
from sagemaker.core.shapes import VpcConfig
@@ -24,6 +24,7 @@
2424
from sagemaker.train.base_trainer import BaseTrainer
2525

2626
from sagemaker.train.evaluate.base_evaluator import BaseEvaluator
27+
from sagemaker.train.evaluate.constants import EvalType
2728

2829

2930
# Test constants
@@ -1840,3 +1841,57 @@ def test_fields_without_spec_type_not_coerced(self):
18401841
{"x": {"default": "5"}}, semantic_values={"x": "5"}
18411842
)
18421843
assert value_map["x"] == "5"
1844+
1845+
1846+
class TestStartExecutionTags:
1847+
"""Tests for merging the ``tags`` field into the evaluation pipeline tags."""
1848+
1849+
@patch("sagemaker.train.common_utils.model_resolution._resolve_base_model")
1850+
def _make_evaluator(self, mock_resolve, mock_session, mock_model_info, tags=None):
1851+
mock_resolve.return_value = mock_model_info
1852+
return BaseEvaluator(
1853+
model=DEFAULT_MODEL,
1854+
s3_output_path=DEFAULT_S3_OUTPUT,
1855+
mlflow_resource_arn=DEFAULT_MLFLOW_ARN,
1856+
model_package_group=DEFAULT_MODEL_PACKAGE_GROUP_ARN,
1857+
sagemaker_session=mock_session,
1858+
tags=tags,
1859+
)
1860+
1861+
def _start(self, evaluator):
1862+
"""Run _start_execution with the pipeline submit mocked out."""
1863+
with (
1864+
patch(
1865+
"sagemaker.train.evaluate.execution.EvaluationPipelineExecution"
1866+
) as mock_execution,
1867+
patch.object(
1868+
BaseEvaluator, "_is_jumpstart_model", new_callable=PropertyMock
1869+
) as mock_is_js,
1870+
):
1871+
mock_is_js.return_value = False
1872+
evaluator._start_execution(
1873+
eval_type=EvalType.BENCHMARK,
1874+
name="test-eval",
1875+
pipeline_definition='{"Steps": []}',
1876+
role_arn=DEFAULT_ROLE_ARN,
1877+
region=DEFAULT_REGION,
1878+
)
1879+
return mock_execution.start.call_args.kwargs["tags"]
1880+
1881+
def test_user_tags_reach_the_pipeline(self, mock_session, mock_model_info):
1882+
"""Tags set on the evaluator must be passed to the pipeline execution."""
1883+
evaluator = self._make_evaluator(
1884+
mock_session=mock_session,
1885+
mock_model_info=mock_model_info,
1886+
tags=[{"key": "sagemaker:project-id", "value": "p-12345"}],
1887+
)
1888+
1889+
assert self._start(evaluator) == [{"key": "sagemaker:project-id", "value": "p-12345"}]
1890+
1891+
def test_no_tags_yields_empty_list(self, mock_session, mock_model_info):
1892+
"""Omitting tags must not fail, and must not invent any tags."""
1893+
evaluator = self._make_evaluator(
1894+
mock_session=mock_session, mock_model_info=mock_model_info, tags=None
1895+
)
1896+
1897+
assert self._start(evaluator) == []

‎sagemaker-train/tests/unit/train/evaluate/test_execution.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,51 @@ def test_create_pipeline_success(self, mock_pipeline_class, mock_get_name, mock_
129129
)
130130
assert result == mock_pipeline
131131

132+
@patch("sagemaker.train.evaluate.execution._get_pipeline_name")
133+
@patch("sagemaker.train.evaluate.execution.Pipeline")
134+
def test_create_pipeline_propagates_user_tags(
135+
self, mock_pipeline_class, mock_get_name, mock_session
136+
):
137+
"""User tags must reach Pipeline.create alongside the evaluation discovery tag."""
138+
mock_get_name.return_value = DEFAULT_PIPELINE_NAME
139+
mock_pipeline_class.create.return_value = MagicMock()
140+
141+
_create_evaluation_pipeline(
142+
eval_type=EvalType.BENCHMARK,
143+
role_arn=DEFAULT_ROLE,
144+
pipeline_definition=DEFAULT_PIPELINE_DEFINITION,
145+
session=mock_session,
146+
region=DEFAULT_REGION,
147+
tags=[{"key": "sagemaker:project-id", "value": "p-12345"}],
148+
)
149+
150+
created_tags = mock_pipeline_class.create.call_args.kwargs["tags"]
151+
pairs = [(t.key, t.value) for t in created_tags]
152+
assert ("sagemaker:project-id", "p-12345") in pairs
153+
# The evaluation discovery tag must still be present.
154+
assert any(key == "SagemakerModelEvaluation" for key, _ in pairs)
155+
156+
@patch("sagemaker.train.evaluate.execution._get_pipeline_name")
157+
@patch("sagemaker.train.evaluate.execution.Pipeline")
158+
def test_create_pipeline_accepts_capitalized_user_tags(
159+
self, mock_pipeline_class, mock_get_name, mock_session
160+
):
161+
"""Capitalized user tags must also be converted into Tag objects."""
162+
mock_get_name.return_value = DEFAULT_PIPELINE_NAME
163+
mock_pipeline_class.create.return_value = MagicMock()
164+
165+
_create_evaluation_pipeline(
166+
eval_type=EvalType.BENCHMARK,
167+
role_arn=DEFAULT_ROLE,
168+
pipeline_definition=DEFAULT_PIPELINE_DEFINITION,
169+
session=mock_session,
170+
region=DEFAULT_REGION,
171+
tags=[{"Key": "sagemaker:project-id", "Value": "p-12345"}],
172+
)
173+
174+
pairs = [(t.key, t.value) for t in mock_pipeline_class.create.call_args.kwargs["tags"]]
175+
assert ("sagemaker:project-id", "p-12345") in pairs
176+
132177
@patch("sagemaker.train.evaluate.execution.Pipeline")
133178
def test_create_pipeline_waits_for_status(self, mock_pipeline_class, mock_session):
134179
"""Test that pipeline waits for active status."""

0 commit comments

Comments
 (0)