Skip to content

Commit 7d0eb92

Browse files
committed
fix(instance-preferences): reject managed spot client-side; make IntPipeVar codegen-stable
The documentation lists managed spot training among the settings that are mutually exclusive with instance_preferences, but the client-side validator only checked instance_type, instance_groups and instance_placement_config, so the spot conflict surfaced as a service error. Both validator copies now reject an explicit enable_managed_spot_training=True; False and unset remain valid. The IntPipeVar annotation on the five instance-preferences count members was a hand edit that generate_shapes could not reproduce: only ResourceConfig.InstanceCount was registered in PIPE_VAR_OVERRIDES, so the next regeneration would have narrowed the others back to Optional[int]. All five are registered, and a unit test asserts the extractor emits IntPipeVar for each so a dropped override fails loudly.
1 parent 3d58dc9 commit 7d0eb92

5 files changed

Lines changed: 91 additions & 11 deletions

File tree

‎sagemaker-core/src/sagemaker/core/modules/utils.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# ANY KIND, either express or implied. See the License for the specific
1212
# language governing permissions and limitations under the License.
1313
"""Utils module."""
14+
1415
from __future__ import absolute_import
1516

1617
import os
@@ -200,7 +201,7 @@ def validate_instance_preferences(compute) -> None:
200201
201202
- instance_preferences is mutually exclusive with the classic
202203
single-cluster fields instance_type / instance_groups /
203-
instance_placement_config.
204+
instance_placement_config, and with managed spot training.
204205
- Instance types must not repeat across preferences.
205206
- Count mode: exactly one of the top-level instance_count (applies to
206207
whichever preference wins) with no per-preference instance_count, or an
@@ -231,6 +232,12 @@ def _value(obj, field):
231232
f"instance_preferences is mutually exclusive with {field}; "
232233
"specify either a single fixed cluster or instance_preferences, not both."
233234
)
235+
# Spot is a bool: only an explicit True conflicts (False/None is the default).
236+
if _value(compute, "enable_managed_spot_training") is True:
237+
raise ValueError(
238+
"instance_preferences is mutually exclusive with managed spot training "
239+
"(enable_managed_spot_training=True)."
240+
)
234241

235242
instance_types = [_value(p, "instance_type") for p in preferences]
236243
duplicates = sorted(

‎sagemaker-core/src/sagemaker/core/tools/constants.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# ANY KIND, either express or implied. See the License for the specific
1212
# language governing permissions and limitations under the License.
1313
"""Constants used in the code_generator modules."""
14+
1415
import os
1516

1617
CLASS_METHODS = set(["create", "add", "register", "import", "list", "get"])
@@ -134,5 +135,16 @@
134135
"InstanceCount": "IntPipeVar",
135136
"VolumeSizeInGB": "IntPipeVar",
136137
"KeepAlivePeriodInSeconds": "IntPipeVar",
138+
"SelectedInstanceCount": "IntPipeVar",
139+
},
140+
"InstancePreference": {
141+
"InstanceCount": "IntPipeVar",
142+
},
143+
"ProcessingClusterConfig": {
144+
"InstanceCount": "IntPipeVar",
145+
"SelectedInstanceCount": "IntPipeVar",
146+
},
147+
"ProcessingInstancePreference": {
148+
"InstanceCount": "IntPipeVar",
137149
},
138150
}

‎sagemaker-core/src/sagemaker/core/training/utils.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ def validate_instance_preferences(compute) -> None:
264264
265265
- instance_preferences is mutually exclusive with the classic
266266
single-cluster fields instance_type / instance_groups /
267-
instance_placement_config.
267+
instance_placement_config, and with managed spot training.
268268
- Instance types must not repeat across preferences.
269269
- Count mode: exactly one of the top-level instance_count (applies to
270270
whichever preference wins) with no per-preference instance_count, or an
@@ -295,6 +295,12 @@ def _value(obj, field):
295295
f"instance_preferences is mutually exclusive with {field}; "
296296
"specify either a single fixed cluster or instance_preferences, not both."
297297
)
298+
# Spot is a bool: only an explicit True conflicts (False/None is the default).
299+
if _value(compute, "enable_managed_spot_training") is True:
300+
raise ValueError(
301+
"instance_preferences is mutually exclusive with managed spot training "
302+
"(enable_managed_spot_training=True)."
303+
)
298304

299305
instance_types = [_value(p, "instance_type") for p in preferences]
300306
duplicates = sorted(

‎sagemaker-core/tests/unit/test_compute_configs.py‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
"""Unit tests for Compute and HyperPodCompute config classes."""
2+
23
import pytest
34
from sagemaker.core.training.configs import Compute, HyperPodCompute
45

@@ -169,6 +170,26 @@ def test_instance_type_rejected_with_preferences(self, compute_cls):
169170
instance_preferences=[InstancePreference(instance_type="ml.m5.xlarge")],
170171
)
171172

173+
def test_managed_spot_rejected_with_preferences(self, compute_cls):
174+
from sagemaker.core.shapes.shapes import InstancePreference
175+
176+
with pytest.raises(ValueError, match="mutually exclusive with managed spot training"):
177+
compute_cls(
178+
instance_count=1,
179+
enable_managed_spot_training=True,
180+
instance_preferences=[InstancePreference(instance_type="ml.m5.xlarge")],
181+
)
182+
183+
def test_managed_spot_false_allowed_with_preferences(self, compute_cls):
184+
from sagemaker.core.shapes.shapes import InstancePreference
185+
186+
compute = compute_cls(
187+
instance_count=1,
188+
enable_managed_spot_training=False,
189+
instance_preferences=[InstancePreference(instance_type="ml.m5.xlarge")],
190+
)
191+
assert compute.enable_managed_spot_training is False
192+
172193
def test_uniform_count_rejected_with_per_preference_counts(self, compute_cls):
173194
from sagemaker.core.shapes.shapes import InstancePreference
174195

@@ -239,9 +260,7 @@ def test_whole_job_plan_allowed_without_per_preference_plans(self, compute_cls):
239260

240261
compute_cls(
241262
instance_count=1,
242-
training_plan_arn=(
243-
"arn:aws:sagemaker:us-west-2:111122223333:training-plan/whole-job"
244-
),
263+
training_plan_arn=("arn:aws:sagemaker:us-west-2:111122223333:training-plan/whole-job"),
245264
instance_preferences=[
246265
InstancePreference(instance_type="ml.p5.48xlarge"),
247266
InstancePreference(instance_type="ml.p4d.24xlarge"),
@@ -260,9 +279,7 @@ def test_modules_compute_instance_preferences_round_trip(self):
260279
InstancePreference(instance_type="ml.p5.48xlarge"),
261280
InstancePreference(instance_type="ml.p4d.24xlarge"),
262281
]
263-
rc = ModulesCompute(
264-
instance_preferences=prefs, instance_count=2
265-
)._to_resource_config()
282+
rc = ModulesCompute(instance_preferences=prefs, instance_count=2)._to_resource_config()
266283
assert [p.instance_type for p in rc.instance_preferences] == [
267284
"ml.p5.48xlarge",
268285
"ml.p4d.24xlarge",
@@ -271,9 +288,7 @@ def test_modules_compute_instance_preferences_round_trip(self):
271288
def test_modules_compute_single_type_still_works(self):
272289
from sagemaker.core.modules.configs import Compute as ModulesCompute
273290

274-
rc = ModulesCompute(
275-
instance_type="ml.m5.xlarge", instance_count=1
276-
)._to_resource_config()
291+
rc = ModulesCompute(instance_type="ml.m5.xlarge", instance_count=1)._to_resource_config()
277292
assert rc.instance_type == "ml.m5.xlarge"
278293

279294

‎sagemaker-core/tests/unit/tools/test_shapes_extractor.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# ANY KIND, either express or implied. See the License for the specific
1212
# language governing permissions and limitations under the License.
1313
"""Unit tests for sagemaker.core.tools.shapes_extractor module."""
14+
1415
from __future__ import absolute_import
1516

1617
import pytest
@@ -343,3 +344,42 @@ def test_get_required_members_none(self, mock_reformat):
343344
result = extractor.get_required_members("TestShape")
344345

345346
assert len(result) == 0
347+
348+
349+
class TestInstancePreferencesPipeVarOverrides:
350+
"""The IntPipeVar annotations on instance-preferences count members come from
351+
PIPE_VAR_OVERRIDES, not the service model. If an override is dropped, codegen
352+
silently narrows the member back to int and pipeline variables stop being
353+
accepted -- so assert the generated type directly."""
354+
355+
@pytest.fixture
356+
def extractor(self, tmp_path):
357+
# Constructing the extractor regenerates shape_dag.py; point that write
358+
# at a temp file so the unit test leaves the checked-in file alone.
359+
with (
360+
patch("sagemaker.core.tools.shapes_extractor.reformat_file_with_black"),
361+
patch(
362+
"sagemaker.core.tools.shapes_extractor.SHAPE_DAG_FILE_PATH",
363+
str(tmp_path / "shape_dag.py"),
364+
),
365+
):
366+
return ShapesExtractor()
367+
368+
@pytest.mark.parametrize(
369+
"shape, member",
370+
[
371+
("ResourceConfig", "instance_count"),
372+
("ResourceConfig", "selected_instance_count"),
373+
("InstancePreference", "instance_count"),
374+
("ProcessingClusterConfig", "instance_count"),
375+
("ProcessingClusterConfig", "selected_instance_count"),
376+
("ProcessingInstancePreference", "instance_count"),
377+
],
378+
)
379+
def test_count_members_generate_as_int_pipe_var(self, extractor, shape, member):
380+
members = extractor.generate_shape_members(shape)
381+
assert member in members, f"{shape}.{member} missing from generated members"
382+
assert "IntPipeVar" in members[member], (
383+
f"{shape}.{member} generated as {members[member]!r}; "
384+
"expected IntPipeVar via PIPE_VAR_OVERRIDES"
385+
)

0 commit comments

Comments
 (0)