Skip to content

Commit 14a9c8d

Browse files
mujtaba1747Syed Jafri
andauthored
fix: filter full recipe template from serverless train( ) (#6021)
* fix: filter full recipe template from serverless train( ) * fix: update hyperparam ref in mtrl trainer * fix: enable only_user_overrides for RLVRTrainer serverless path * chore: removing extra logging * testing: added new integ test for nemotron * fix: use self.compute instead of user override flag for apply_recipe_hyperparams fix * chore: undo minor indent changes in trainers * code cleanup: recipe override tests --------- Co-authored-by: Syed Jafri <syedjfr@amazon.com>
1 parent 9831ab2 commit 14a9c8d

3 files changed

Lines changed: 373 additions & 3 deletions

File tree

‎sagemaker-train/src/sagemaker/train/base_trainer.py‎

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,54 @@ def _patch_resolved_recipe(self, resolved: Dict[str, Any]) -> None:
236236
if dotpath:
237237
_set_nested_value(resolved, dotpath, value)
238238

239-
def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]) -> Dict[str, Any]:
239+
def _get_user_provided_recipe_keys(self) -> set:
240+
"""Return the set of leaf keys the user explicitly provided.
241+
242+
Collects keys from every source that represents an explicit user choice:
243+
244+
* direct hyperparameter assignments (``trainer.hyperparameters.x = val``),
245+
tracked in ``hyperparameters._user_set``;
246+
* the programmatic ``overrides`` dict passed at construction;
247+
* the user recipe YAML file passed at construction.
248+
249+
These are the keys that should be forwarded to a serverless training job,
250+
alongside the Hub override spec. Full recipe-template internal keys that
251+
the user never touched are intentionally excluded.
252+
253+
Returns:
254+
Set of leaf key names the user explicitly provided. Empty when the
255+
user provided nothing beyond Hub defaults.
256+
"""
257+
keys: set = set()
258+
259+
# Direct hyperparameter assignments (always members of the Hub spec).
260+
user_set = getattr(getattr(self, 'hyperparameters', None), '_user_set', None)
261+
if isinstance(user_set, set):
262+
keys.update(user_set)
263+
264+
# Programmatic overrides dict (may contain non-spec recipe keys).
265+
overrides = getattr(self, '_overrides', None)
266+
if isinstance(overrides, dict) and overrides:
267+
keys.update(flatten_resolved_recipe(overrides).keys())
268+
269+
# User recipe YAML file (may contain non-spec recipe keys).
270+
recipe_path = getattr(self, '_recipe_path', None)
271+
if recipe_path:
272+
try:
273+
from sagemaker.train.recipe_resolver import _load_user_recipe
274+
275+
user_recipe = _load_user_recipe(recipe_path)
276+
keys.update(flatten_resolved_recipe(user_recipe).keys())
277+
except Exception as e: # pragma: no cover - best-effort key discovery
278+
logger.debug("Could not load user recipe to collect keys: %s", e)
279+
280+
return keys
281+
282+
283+
def _apply_recipe_to_hyperparameters(
284+
self,
285+
final_hyperparameters: Dict[str, Any],
286+
) -> Dict[str, Any]:
240287
"""Apply resolved recipe values to final_hyperparameters dict.
241288
242289
If recipe/overrides were provided, or if the user set hyperparameters
@@ -247,6 +294,10 @@ def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]
247294
Values are converted to strings (matching the SageMaker API
248295
expectation for hyperparameter values).
249296
297+
For serverless training (``self.compute`` is None), only user-provided
298+
keys (from .hyperparameters.*, recipe or overrides dict) are included because CreateTrainingJob limits HyperParameters to
299+
100 members and the full resolved recipe can exceed that.
300+
250301
Args:
251302
final_hyperparameters: The hyperparameters dict from to_dict().
252303
@@ -259,12 +310,30 @@ def _apply_recipe_to_hyperparameters(self, final_hyperparameters: Dict[str, Any]
259310
try:
260311
resolved = self.get_resolved_recipe()
261312
except NoRecipeError:
313+
262314
return final_hyperparameters
263315

264316
flat = flatten_resolved_recipe(resolved)
317+
318+
# Serverless (compute is None) → only user-provided keys + defaults;
319+
allowed_keys = None
320+
if getattr(self, 'compute', None) is None:
321+
try:
322+
allowed_keys = self._get_user_provided_recipe_keys()
323+
except Exception as e:
324+
logger.warning(
325+
"Failed to determine user-provided recipe keys (%s); "
326+
"falling back to submitting the full resolved recipe.",
327+
e,
328+
)
329+
allowed_keys = None
330+
265331
for k, v in flat.items():
266-
if v is not None:
267-
final_hyperparameters[k] = str(v) if not isinstance(v, str) else v
332+
if v is None:
333+
continue
334+
if allowed_keys is not None and k not in allowed_keys:
335+
continue
336+
final_hyperparameters[k] = str(v) if not isinstance(v, str) else v
268337

269338
return final_hyperparameters
270339

‎sagemaker-train/tests/integ/train/test_recipe_override_integration.py‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,21 @@
1313
"""Integration tests for recipe override feature (get_resolved_recipe)."""
1414
from __future__ import absolute_import
1515

16+
import logging
1617
import os
1718
import tempfile
1819
import time
1920

2021
import pytest
2122
import yaml
2223

24+
logger = logging.getLogger(__name__)
25+
2326
from sagemaker.train.sft_trainer import SFTTrainer
27+
from sagemaker.train.rlvr_trainer import RLVRTrainer
2428
from sagemaker.train.common import TrainingType
29+
from sagemaker.train.recipe_resolver import flatten_resolved_recipe
30+
from sagemaker.core.training.configs import TrainingJobCompute
2531

2632

2733
# Ensure bundled service model is available for botocore
@@ -1002,3 +1008,225 @@ def test_model_trainer_get_resolved_recipe_is_idempotent(self):
10021008

10031009
finally:
10041010
os.unlink(recipe_path)
1011+
1012+
1013+
class TestRLVRServerlessOnlyUserOverrideKeys:
1014+
"""Integration tests for serverless path filtering (compute=None excludes non-overridden keys)."""
1015+
1016+
def test_rlvr_serverless_only_user_override_keys_applied(self, sagemaker_session):
1017+
"""Test that serverless path (compute=None) excludes non-overridden recipe keys.
1018+
1019+
When compute is None, CreateTrainingJob limits HyperParameters to 100 members.
1020+
A full recipe template can have several hundred internal leaf keys. This test
1021+
verifies that after applying overrides, only the user-provided keys from
1022+
overrides/recipe/direct assignment appear — non-overridden recipe template
1023+
keys must NOT leak into the result.
1024+
"""
1025+
recipe_content = {
1026+
"training_config": {
1027+
"data": {
1028+
"max_prompt_length": 3070,
1029+
},
1030+
}
1031+
}
1032+
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
1033+
yaml.dump(recipe_content, f)
1034+
recipe_path = f.name
1035+
1036+
try:
1037+
rlvr_trainer = RLVRTrainer(
1038+
model="huggingface-reasoning-nvidia-nemotron-3-nano-30b-a3b-bf16",
1039+
model_package_group="sdk-test-finetuned-models",
1040+
training_dataset="s3://mc-flows-sdk-testing/input_data/rlvr-rlaif-test-data/train_285.jsonl",
1041+
s3_output_path="s3://mc-flows-sdk-testing/output/",
1042+
sagemaker_session=sagemaker_session,
1043+
custom_reward_function="arn:aws:sagemaker:us-west-2:729646638167:hub-content/sdktest/JsonDoc/rlvr-test-rf/0.0.1",
1044+
accept_eula=True,
1045+
base_job_name="rlvr-override-keys-integ",
1046+
overrides={
1047+
"training_config": {
1048+
"learning_rate": 2e-5,
1049+
"max_epochs": 10,
1050+
"train_val_split_ratio": 0.8,
1051+
"temperature": 1.1,
1052+
}
1053+
},
1054+
recipe=recipe_path,
1055+
)
1056+
1057+
rlvr_trainer.hyperparameters.use_kl_loss = True
1058+
rlvr_trainer.hyperparameters.kl_loss_coef = 0.05
1059+
1060+
# Get baseline hyperparameters (the Hub spec defaults)
1061+
baseline_hp = rlvr_trainer.hyperparameters.to_dict()
1062+
1063+
# Serverless path (compute=None): only user-provided keys applied
1064+
assert rlvr_trainer.compute is None
1065+
result_hp = rlvr_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy())
1066+
1067+
# Simulate serverful path (compute set): full recipe applied
1068+
rlvr_trainer.compute = TrainingJobCompute(instance_type="ml.p5.48xlarge", instance_count=1)
1069+
full_hp = rlvr_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy())
1070+
rlvr_trainer.compute = None # reset
1071+
1072+
logger.info(f"Baseline HP keys: {len(baseline_hp)}")
1073+
logger.info(f"User-override-only HP keys: {len(result_hp)}")
1074+
logger.info(f"Full recipe HP keys: {len(full_hp)}")
1075+
1076+
# Keys the user explicitly provided
1077+
expected_override_keys = {"learning_rate", "max_epochs", "train_val_split_ratio", "temperature"}
1078+
expected_recipe_keys = {"max_prompt_length"}
1079+
expected_direct_hp_keys = {"use_kl_loss", "kl_loss_coef"}
1080+
all_user_keys = expected_override_keys | expected_recipe_keys | expected_direct_hp_keys
1081+
1082+
# All user-provided keys must be present in the user-override result
1083+
for key in all_user_keys:
1084+
assert key in result_hp, (
1085+
f"User-provided key '{key}' missing from serverless (compute=None) result"
1086+
)
1087+
1088+
# Dynamically compute recipe keys NOT in the override spec by fetching
1089+
# the full resolved recipe and subtracting the spec keys + user-provided keys.
1090+
# Only consider keys with non-None values (None keys are skipped by _apply_recipe).
1091+
resolved_recipe = rlvr_trainer.get_resolved_recipe()
1092+
flat_recipe = flatten_resolved_recipe(resolved_recipe)
1093+
all_recipe_keys = {k for k, v in flat_recipe.items() if v is not None}
1094+
override_spec_keys = set(rlvr_trainer.hyperparameters._specs.keys())
1095+
recipe_internal_keys_not_in_spec = all_recipe_keys - override_spec_keys - all_user_keys
1096+
1097+
logger.info(f"All recipe keys from Hub ({len(all_recipe_keys)}): {sorted(all_recipe_keys)}")
1098+
logger.info(f"Override spec keys ({len(override_spec_keys)}): {sorted(override_spec_keys)}")
1099+
logger.info(
1100+
f"Recipe internal keys NOT in spec ({len(recipe_internal_keys_not_in_spec)}): "
1101+
f"{sorted(recipe_internal_keys_not_in_spec)}"
1102+
)
1103+
1104+
assert len(recipe_internal_keys_not_in_spec) > 0, (
1105+
"Expected recipe template to have keys beyond the override spec, but found none."
1106+
)
1107+
1108+
# These internal recipe keys must NOT appear in the serverless result
1109+
leaked_keys = recipe_internal_keys_not_in_spec & set(result_hp.keys())
1110+
assert not leaked_keys, (
1111+
f"Non-overridable recipe template keys leaked into the serverless hyperparameters: "
1112+
f"{sorted(leaked_keys)}. These keys are not in the override spec and should be "
1113+
f"excluded when compute=None (serverless)."
1114+
)
1115+
1116+
# The full recipe (compute set) MUST contain these internal keys
1117+
missing_from_full = recipe_internal_keys_not_in_spec - set(full_hp.keys())
1118+
assert not missing_from_full, (
1119+
f"Full recipe (compute set) is missing expected internal keys: "
1120+
f"{sorted(missing_from_full)}. The full recipe path should include all template keys."
1121+
)
1122+
1123+
# The user-override result should not exceed the 100-key CreateTrainingJob limit
1124+
assert len(result_hp) <= 100, (
1125+
f"Serverless hyperparameters have {len(result_hp)} keys, exceeding the "
1126+
f"CreateTrainingJob limit of 100. Non-overridden recipe keys are leaking through."
1127+
)
1128+
1129+
finally:
1130+
os.unlink(recipe_path)
1131+
1132+
def test_sft_nova_serverless_only_user_override_keys_applied(self, sagemaker_session_us_east_1):
1133+
"""Test that Nova SFT serverless path (compute=None) excludes non-overridden recipe keys.
1134+
1135+
Same principle as the RLVR Nemotron test: when compute is None,
1136+
internal recipe template keys that the user never touched must not appear
1137+
in the hyperparameters sent to CreateTrainingJob.
1138+
"""
1139+
sft_trainer = SFTTrainer(
1140+
model="nova-textgeneration-lite-v2",
1141+
training_type=TrainingType.LORA,
1142+
model_package_group="sdk-test-finetuned-models",
1143+
training_dataset="s3://sagemaker-us-east-1-784379639078/input_data/sft-nova/sft_200_samples.jsonl",
1144+
s3_output_path="s3://sagemaker-us-east-1-784379639078/output/",
1145+
sagemaker_session=sagemaker_session_us_east_1,
1146+
accept_eula=True,
1147+
base_job_name="sft-nova-override-keys-integ",
1148+
overrides={
1149+
"training_config": {
1150+
"learning_rate": 3e-5,
1151+
"max_steps": 50,
1152+
}
1153+
},
1154+
)
1155+
1156+
sft_trainer.hyperparameters.warmup_steps = 5
1157+
sft_trainer.hyperparameters.weight_decay = 0.01
1158+
1159+
baseline_hp = sft_trainer.hyperparameters.to_dict()
1160+
1161+
# Serverless path (compute=None): only user-provided keys applied
1162+
assert sft_trainer.compute is None
1163+
result_hp = sft_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy())
1164+
1165+
# Simulate serverful path (compute set): full recipe applied
1166+
sft_trainer.compute = TrainingJobCompute(instance_type="ml.p5.48xlarge", instance_count=1)
1167+
full_hp = sft_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy())
1168+
sft_trainer.compute = None # reset
1169+
1170+
logger.info(f"Nova SFT — Baseline HP keys: {len(baseline_hp)}")
1171+
logger.info(f"Nova SFT — User-override-only HP keys: {len(result_hp)}")
1172+
logger.info(f"Nova SFT — Full recipe HP keys: {len(full_hp)}")
1173+
1174+
# Keys the user explicitly provided
1175+
expected_override_keys = {"learning_rate", "max_steps"}
1176+
expected_direct_hp_keys = {"warmup_steps", "weight_decay"}
1177+
all_user_keys = expected_override_keys | expected_direct_hp_keys
1178+
1179+
for key in all_user_keys:
1180+
assert key in result_hp, (
1181+
f"User-provided key '{key}' missing from serverless (compute=None) result"
1182+
)
1183+
1184+
# Full recipe should have more keys than the user-override-only result
1185+
full_only_keys = set(full_hp.keys()) - set(result_hp.keys())
1186+
assert len(full_only_keys) > 0, (
1187+
"Full recipe should contain additional keys beyond the user-override-only result."
1188+
)
1189+
logger.info(
1190+
f"Nova SFT — Keys excluded from serverless path: "
1191+
f"{len(full_only_keys)} keys — {sorted(list(full_only_keys))}"
1192+
)
1193+
1194+
# Dynamically compute recipe keys NOT in the override spec by fetching
1195+
# the full resolved recipe and subtracting the spec keys + user-provided keys.
1196+
# Only consider keys with non-None values (None keys are skipped by _apply_recipe).
1197+
resolved_recipe = sft_trainer.get_resolved_recipe()
1198+
flat_recipe = flatten_resolved_recipe(resolved_recipe)
1199+
all_recipe_keys = {k for k, v in flat_recipe.items() if v is not None}
1200+
override_spec_keys = set(sft_trainer.hyperparameters._specs.keys())
1201+
recipe_internal_keys_not_in_spec = all_recipe_keys - override_spec_keys - all_user_keys
1202+
1203+
logger.info(f"Nova SFT — All recipe keys from Hub ({len(all_recipe_keys)}): {sorted(all_recipe_keys)}")
1204+
logger.info(f"Nova SFT — Override spec keys ({len(override_spec_keys)}): {sorted(override_spec_keys)}")
1205+
logger.info(
1206+
f"Nova SFT — Recipe internal keys NOT in spec ({len(recipe_internal_keys_not_in_spec)}): "
1207+
f"{sorted(recipe_internal_keys_not_in_spec)}"
1208+
)
1209+
1210+
assert len(recipe_internal_keys_not_in_spec) > 0, (
1211+
"Expected Nova recipe template to have keys beyond the override spec, but found none."
1212+
)
1213+
1214+
# These internal recipe keys must NOT appear in the serverless result
1215+
leaked_keys = recipe_internal_keys_not_in_spec & set(result_hp.keys())
1216+
assert not leaked_keys, (
1217+
f"Non-overridable Nova recipe template keys leaked into serverless hyperparameters: "
1218+
f"{sorted(leaked_keys)}. These keys are not in the override spec and should be "
1219+
f"excluded when compute=None (serverless)."
1220+
)
1221+
1222+
# The full recipe (compute set) MUST contain these internal keys
1223+
missing_from_full = recipe_internal_keys_not_in_spec - set(full_hp.keys())
1224+
assert not missing_from_full, (
1225+
f"Full recipe (compute set) is missing expected internal keys: "
1226+
f"{sorted(missing_from_full)}. The full recipe path should include all template keys."
1227+
)
1228+
1229+
assert len(result_hp) <= 100, (
1230+
f"Serverless hyperparameters have {len(result_hp)} keys, exceeding the "
1231+
f"CreateTrainingJob limit of 100."
1232+
)

0 commit comments

Comments
 (0)