Skip to content

Commit b8b4518

Browse files
mohamedzeidan2021Mohamed Zeidan
andauthored
fix(train): correct Networking field names in ModelTrainer intelligent defaults (#6064)
Fixes two bugs in _populate_intelligent_defaults_from_training_job_space() in model_trainer.py that prevent ModelTrainer from working when sagemaker_config has a VpcConfig set. Bug 1: Networking() was passed default_enable_network_isolation, which is not a field in sagemaker-core 2.x. The correct field is enable_network_isolation. This raised a pydantic ValidationError on every train() call when VPC config was present. Bug 2: When updating an existing Networking object, security_group_ids was mistakenly assigned the subnets value from TRAINING_JOB_SUBNETS_PATH instead of security_group_ids from TRAINING_JOB_SECURITY_GROUP_IDS_PATH. Adds unit tests covering both bug paths (fresh Networking construction and updating an existing Networking object). Fix originally from #5866; unit tests added here. Co-authored-by: Mohamed Zeidan <zeidmo@amazon.com>
1 parent 72fb2ed commit b8b4518

2 files changed

Lines changed: 118 additions & 3 deletions

File tree

‎sagemaker-train/src/sagemaker/train/model_trainer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ def _populate_intelligent_defaults_from_training_job_space(self):
331331
if not self.networking:
332332
if default_enable_network_isolation is not None or default_vpc_config is not None:
333333
self.networking = Networking(
334-
default_enable_network_isolation=default_enable_network_isolation,
334+
enable_network_isolation=default_enable_network_isolation,
335335
subnets=self.config_mgr.resolve_value_from_config(
336336
config_path=TRAINING_JOB_SUBNETS_PATH
337337
),
@@ -347,8 +347,8 @@ def _populate_intelligent_defaults_from_training_job_space(self):
347347
config_path=TRAINING_JOB_SUBNETS_PATH
348348
)
349349
if self.networking.security_group_ids is None:
350-
self.networking.subnets = self.config_mgr.resolve_value_from_config(
351-
config_path=TRAINING_JOB_SUBNETS_PATH
350+
self.networking.security_group_ids = self.config_mgr.resolve_value_from_config(
351+
config_path=TRAINING_JOB_SECURITY_GROUP_IDS_PATH
352352
)
353353

354354
if not self.output_data_config:

‎sagemaker-train/tests/unit/train/test_model_trainer.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,15 @@
3434
MODEL_TRAINER,
3535
_simple_path,
3636
TRAINING_JOB_RESOURCE_CONFIG_PATH,
37+
TRAINING_JOB_ENABLE_NETWORK_ISOLATION_PATH,
38+
TRAINING_JOB_VPC_CONFIG_PATH,
39+
TRAINING_JOB_SUBNETS_PATH,
40+
TRAINING_JOB_SECURITY_GROUP_IDS_PATH,
3741
SAGEMAKER,
3842
PYTHON_SDK,
3943
MODULES,
4044
)
45+
from sagemaker.core.config.config_manager import SageMakerConfig
4146
from sagemaker.core.helper.session_helper import Session
4247
from sagemaker.train.model_trainer import ModelTrainer, Mode
4348
from sagemaker.train.defaults import DEFAULT_INSTANCE_TYPE
@@ -1756,3 +1761,113 @@ def test_resolve_staging_bucket_returns_default_on_iam_error(model_trainer):
17561761

17571762
assert bucket == DEFAULT_BUCKET
17581763
assert prefix is None
1764+
1765+
1766+
# Networking intelligent defaults from the Training Job config space. These guard the two
1767+
# bugs fixed in _populate_intelligent_defaults_from_training_job_space: (1) the Networking
1768+
# constructor must be given `enable_network_isolation` (not the non-existent
1769+
# `default_enable_network_isolation`), and (2) an existing Networking object with an unset
1770+
# `security_group_ids` must be filled from the security-group-ids path, not the subnets path.
1771+
1772+
NETWORKING_DEFAULT_SUBNETS = ["subnet-000000000000"]
1773+
NETWORKING_DEFAULT_SECURITY_GROUP_IDS = ["sg-000000000000"]
1774+
1775+
1776+
def _make_config_mgr(values):
1777+
"""Return a real SageMakerConfig whose resolve_value_from_config is keyed by config_path.
1778+
1779+
A real instance (rather than a bare MagicMock) is required because ModelTrainer validates
1780+
that config_mgr is a SageMakerConfig on assignment.
1781+
"""
1782+
config_mgr = SageMakerConfig()
1783+
config_mgr.resolve_value_from_config = MagicMock(
1784+
side_effect=lambda **kwargs: values.get(kwargs["config_path"])
1785+
)
1786+
return config_mgr
1787+
1788+
1789+
def test_networking_intelligent_defaults_creates_networking(model_trainer):
1790+
"""A VpcConfig in the config space builds a valid Networking with the correct fields.
1791+
1792+
Regression for Bug 1: passing `default_enable_network_isolation` to Networking() raised
1793+
a pydantic ValidationError because that field does not exist.
1794+
"""
1795+
model_trainer.networking = None
1796+
model_trainer.config_mgr = _make_config_mgr(
1797+
{
1798+
TRAINING_JOB_ENABLE_NETWORK_ISOLATION_PATH: True,
1799+
TRAINING_JOB_VPC_CONFIG_PATH: {
1800+
"subnets": NETWORKING_DEFAULT_SUBNETS,
1801+
"security_group_ids": NETWORKING_DEFAULT_SECURITY_GROUP_IDS,
1802+
},
1803+
TRAINING_JOB_SUBNETS_PATH: NETWORKING_DEFAULT_SUBNETS,
1804+
TRAINING_JOB_SECURITY_GROUP_IDS_PATH: NETWORKING_DEFAULT_SECURITY_GROUP_IDS,
1805+
}
1806+
)
1807+
1808+
model_trainer._populate_intelligent_defaults_from_training_job_space()
1809+
1810+
assert model_trainer.networking is not None
1811+
assert model_trainer.networking.enable_network_isolation is True
1812+
assert model_trainer.networking.subnets == NETWORKING_DEFAULT_SUBNETS
1813+
assert model_trainer.networking.security_group_ids == NETWORKING_DEFAULT_SECURITY_GROUP_IDS
1814+
1815+
1816+
def test_networking_intelligent_defaults_no_vpc_config_leaves_networking_unset(model_trainer):
1817+
"""No network isolation and no VpcConfig means Networking is not created."""
1818+
model_trainer.networking = None
1819+
model_trainer.config_mgr = _make_config_mgr({})
1820+
1821+
model_trainer._populate_intelligent_defaults_from_training_job_space()
1822+
1823+
assert model_trainer.networking is None
1824+
1825+
1826+
def test_networking_intelligent_defaults_fills_security_group_ids_on_existing(model_trainer):
1827+
"""An existing Networking with unset security_group_ids is filled from the SG path.
1828+
1829+
Regression for Bug 2: the missing security_group_ids branch mistakenly assigned to
1830+
`subnets` using the subnets path, so security_group_ids was never populated.
1831+
"""
1832+
model_trainer.networking = Networking(
1833+
enable_network_isolation=False,
1834+
subnets=["subnet-preexisting"],
1835+
security_group_ids=None,
1836+
)
1837+
model_trainer.config_mgr = _make_config_mgr(
1838+
{
1839+
TRAINING_JOB_SUBNETS_PATH: NETWORKING_DEFAULT_SUBNETS,
1840+
TRAINING_JOB_SECURITY_GROUP_IDS_PATH: NETWORKING_DEFAULT_SECURITY_GROUP_IDS,
1841+
}
1842+
)
1843+
1844+
model_trainer._populate_intelligent_defaults_from_training_job_space()
1845+
1846+
# security_group_ids is filled from the security-group-ids path...
1847+
assert model_trainer.networking.security_group_ids == NETWORKING_DEFAULT_SECURITY_GROUP_IDS
1848+
# ...and the pre-existing subnets are preserved (not overwritten).
1849+
assert model_trainer.networking.subnets == ["subnet-preexisting"]
1850+
model_trainer.config_mgr.resolve_value_from_config.assert_any_call(
1851+
config_path=TRAINING_JOB_SECURITY_GROUP_IDS_PATH
1852+
)
1853+
1854+
1855+
def test_networking_intelligent_defaults_fills_subnets_on_existing(model_trainer):
1856+
"""An existing Networking with unset subnets is filled from the subnets path."""
1857+
model_trainer.networking = Networking(
1858+
enable_network_isolation=False,
1859+
subnets=None,
1860+
security_group_ids=["sg-preexisting"],
1861+
)
1862+
model_trainer.config_mgr = _make_config_mgr(
1863+
{
1864+
TRAINING_JOB_SUBNETS_PATH: NETWORKING_DEFAULT_SUBNETS,
1865+
TRAINING_JOB_SECURITY_GROUP_IDS_PATH: NETWORKING_DEFAULT_SECURITY_GROUP_IDS,
1866+
}
1867+
)
1868+
1869+
model_trainer._populate_intelligent_defaults_from_training_job_space()
1870+
1871+
assert model_trainer.networking.subnets == NETWORKING_DEFAULT_SUBNETS
1872+
# pre-existing security_group_ids are preserved.
1873+
assert model_trainer.networking.security_group_ids == ["sg-preexisting"]

0 commit comments

Comments
 (0)