Skip to content

Commit f883b2f

Browse files
committed
fix(instance-preferences): reject preference lists on recipe trainers with a clear error
Recipe-based trainers (SFTTrainer, DPOTrainer, RLVRTrainer) accept a Compute for serverful training, and Compute now carries instance_preferences. Recipes are rendered for one instance type: the device class selects the image and launcher, and the type is validated against the model's allowed-type list. A service-chosen type cannot apply, so instance_preferences is unsupported on this path, as documented. Today the unsupported case surfaces indirectly. With no instance_type set, the allowed-type check reports "Instance type 'None' is not supported" before ModelTrainer.from_recipe can raise its explicit message. _train_serverful_smtj now rejects a preference list up front, before the recipe fetch, with the same wording from_recipe uses. Two existing tests modelled Compute as a bare MagicMock, whose auto-created instance_preferences attribute is truthy; they now pin it to None, as a real Compute without preferences reports.
1 parent 62dca3b commit f883b2f

4 files changed

Lines changed: 47 additions & 0 deletions

File tree

‎sagemaker-train/src/sagemaker/train/base_trainer.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,15 @@ def _train_serverful_smtj(self, training_dataset=None, validation_dataset=None,
901901
role = self.role
902902

903903
compute = self.compute
904+
# Recipes are rendered for one instance type (device class, image,
905+
# launcher), so a service-chosen type cannot apply. Fail before the
906+
# recipe fetch, and before the type-enum check reports a confusing
907+
# "Instance type 'None' is not supported".
908+
if getattr(compute, "instance_preferences", None):
909+
raise ValueError(
910+
"Training recipes do not support ``instance_preferences``. "
911+
"Set a single ``instance_type`` in Compute when using a recipe-based trainer."
912+
)
904913
customization_technique = self._customization_technique
905914

906915
# Resolve the recipe S3 URI from hub metadata

‎sagemaker-train/tests/unit/train/test_base_trainer_compute.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ def test_compute_attributes_forwarded(self):
112112
trainer = _ConcreteTrainer()
113113
trainer.compute = MagicMock(
114114
instance_type="ml.p4d.24xlarge",
115+
instance_preferences=None,
115116
instance_count=4,
116117
volume_size_in_gb=300,
117118
keep_alive_period_in_seconds=1200,
@@ -131,6 +132,7 @@ def test_training_plan_arn_forwarded(self):
131132
trainer = _ConcreteTrainer()
132133
trainer.compute = MagicMock(
133134
instance_type="ml.p5.48xlarge",
135+
instance_preferences=None,
134136
instance_count=2,
135137
volume_size_in_gb=500,
136138
keep_alive_period_in_seconds=0,
@@ -147,6 +149,7 @@ def test_training_plan_arn_none_when_not_set(self):
147149
trainer = _ConcreteTrainer()
148150
trainer.compute = MagicMock(
149151
instance_type="ml.p4d.24xlarge",
152+
instance_preferences=None,
150153
instance_count=1,
151154
volume_size_in_gb=30,
152155
keep_alive_period_in_seconds=0,

‎sagemaker-train/tests/unit/train/test_base_trainer_serverful.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ def __init__(self, **kwargs):
2929
self.compute = MagicMock(
3030
instance_type="ml.p4d.24xlarge",
3131
instance_count=1,
32+
instance_preferences=None,
3233
volume_size_in_gb=100,
3334
keep_alive_period_in_seconds=None,
3435
training_plan_arn=None,
@@ -388,3 +389,36 @@ def test_skips_validation_when_enum_unavailable(self, mock_enum):
388389

389390
# Any instance type is accepted; the method returns None to signal skip.
390391
assert trainer._validate_instance_type("ml.g5.xlarge", MagicMock()) is None
392+
393+
394+
class TestInstancePreferencesRejected:
395+
"""Recipes are rendered for one instance type, so a preference list must be
396+
refused up front -- before the recipe fetch, and before the instance-type
397+
enum check can report a misleading "Instance type 'None' is not supported"."""
398+
399+
@pytest.fixture
400+
def trainer(self):
401+
trainer = _ConcreteTrainer()
402+
trainer.compute = MagicMock(
403+
instance_type=None,
404+
instance_count=2,
405+
instance_preferences=[MagicMock(instance_type="ml.p5.48xlarge")],
406+
)
407+
return trainer
408+
409+
def test_rejected_before_any_recipe_or_network_access(self, trainer):
410+
with patch("sagemaker.train.base_trainer.get_recipe_s3_uri") as fetch, pytest.raises(
411+
ValueError, match="Training recipes do not support ``instance_preferences``"
412+
):
413+
trainer._train_serverful_smtj(training_dataset="s3://bucket/train.jsonl")
414+
fetch.assert_not_called()
415+
416+
def test_message_names_the_fix(self, trainer):
417+
with pytest.raises(ValueError, match="Set a single ``instance_type`` in Compute"):
418+
trainer._train_serverful_smtj(training_dataset="s3://bucket/train.jsonl")
419+
420+
def test_single_instance_type_is_unaffected(self):
421+
trainer = _ConcreteTrainer()
422+
_, kwargs = _run_serverful(trainer)
423+
assert kwargs["compute"].instance_type == "ml.p4d.24xlarge"
424+
assert not kwargs["compute"].instance_preferences

‎sagemaker-train/tests/unit/train/test_serverful_recipe_validation.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ def __init__(self, **kwargs):
3838
self.validation_dataset = None
3939
self.compute = MagicMock(
4040
instance_type="ml.p5.48xlarge",
41+
instance_preferences=None,
4142
instance_count=4,
4243
volume_size_in_gb=100,
4344
keep_alive_period_in_seconds=None,

0 commit comments

Comments
 (0)