|
13 | 13 | """Integration tests for recipe override feature (get_resolved_recipe).""" |
14 | 14 | from __future__ import absolute_import |
15 | 15 |
|
| 16 | +import logging |
16 | 17 | import os |
17 | 18 | import tempfile |
18 | 19 | import time |
19 | 20 |
|
20 | 21 | import pytest |
21 | 22 | import yaml |
22 | 23 |
|
| 24 | +logger = logging.getLogger(__name__) |
| 25 | + |
23 | 26 | from sagemaker.train.sft_trainer import SFTTrainer |
| 27 | +from sagemaker.train.rlvr_trainer import RLVRTrainer |
24 | 28 | from sagemaker.train.common import TrainingType |
| 29 | +from sagemaker.train.recipe_resolver import flatten_resolved_recipe |
| 30 | +from sagemaker.core.training.configs import TrainingJobCompute |
25 | 31 |
|
26 | 32 |
|
27 | 33 | # Ensure bundled service model is available for botocore |
@@ -1002,3 +1008,225 @@ def test_model_trainer_get_resolved_recipe_is_idempotent(self): |
1002 | 1008 |
|
1003 | 1009 | finally: |
1004 | 1010 | os.unlink(recipe_path) |
| 1011 | + |
| 1012 | + |
| 1013 | +class TestRLVRServerlessOnlyUserOverrideKeys: |
| 1014 | + """Integration tests for serverless path filtering (compute=None excludes non-overridden keys).""" |
| 1015 | + |
| 1016 | + def test_rlvr_serverless_only_user_override_keys_applied(self, sagemaker_session): |
| 1017 | + """Test that serverless path (compute=None) excludes non-overridden recipe keys. |
| 1018 | +
|
| 1019 | + When compute is None, CreateTrainingJob limits HyperParameters to 100 members. |
| 1020 | + A full recipe template can have several hundred internal leaf keys. This test |
| 1021 | + verifies that after applying overrides, only the user-provided keys from |
| 1022 | + overrides/recipe/direct assignment appear — non-overridden recipe template |
| 1023 | + keys must NOT leak into the result. |
| 1024 | + """ |
| 1025 | + recipe_content = { |
| 1026 | + "training_config": { |
| 1027 | + "data": { |
| 1028 | + "max_prompt_length": 3070, |
| 1029 | + }, |
| 1030 | + } |
| 1031 | + } |
| 1032 | + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: |
| 1033 | + yaml.dump(recipe_content, f) |
| 1034 | + recipe_path = f.name |
| 1035 | + |
| 1036 | + try: |
| 1037 | + rlvr_trainer = RLVRTrainer( |
| 1038 | + model="huggingface-reasoning-nvidia-nemotron-3-nano-30b-a3b-bf16", |
| 1039 | + model_package_group="sdk-test-finetuned-models", |
| 1040 | + training_dataset="s3://mc-flows-sdk-testing/input_data/rlvr-rlaif-test-data/train_285.jsonl", |
| 1041 | + s3_output_path="s3://mc-flows-sdk-testing/output/", |
| 1042 | + sagemaker_session=sagemaker_session, |
| 1043 | + custom_reward_function="arn:aws:sagemaker:us-west-2:729646638167:hub-content/sdktest/JsonDoc/rlvr-test-rf/0.0.1", |
| 1044 | + accept_eula=True, |
| 1045 | + base_job_name="rlvr-override-keys-integ", |
| 1046 | + overrides={ |
| 1047 | + "training_config": { |
| 1048 | + "learning_rate": 2e-5, |
| 1049 | + "max_epochs": 10, |
| 1050 | + "train_val_split_ratio": 0.8, |
| 1051 | + "temperature": 1.1, |
| 1052 | + } |
| 1053 | + }, |
| 1054 | + recipe=recipe_path, |
| 1055 | + ) |
| 1056 | + |
| 1057 | + rlvr_trainer.hyperparameters.use_kl_loss = True |
| 1058 | + rlvr_trainer.hyperparameters.kl_loss_coef = 0.05 |
| 1059 | + |
| 1060 | + # Get baseline hyperparameters (the Hub spec defaults) |
| 1061 | + baseline_hp = rlvr_trainer.hyperparameters.to_dict() |
| 1062 | + |
| 1063 | + # Serverless path (compute=None): only user-provided keys applied |
| 1064 | + assert rlvr_trainer.compute is None |
| 1065 | + result_hp = rlvr_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy()) |
| 1066 | + |
| 1067 | + # Simulate serverful path (compute set): full recipe applied |
| 1068 | + rlvr_trainer.compute = TrainingJobCompute(instance_type="ml.p5.48xlarge", instance_count=1) |
| 1069 | + full_hp = rlvr_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy()) |
| 1070 | + rlvr_trainer.compute = None # reset |
| 1071 | + |
| 1072 | + logger.info(f"Baseline HP keys: {len(baseline_hp)}") |
| 1073 | + logger.info(f"User-override-only HP keys: {len(result_hp)}") |
| 1074 | + logger.info(f"Full recipe HP keys: {len(full_hp)}") |
| 1075 | + |
| 1076 | + # Keys the user explicitly provided |
| 1077 | + expected_override_keys = {"learning_rate", "max_epochs", "train_val_split_ratio", "temperature"} |
| 1078 | + expected_recipe_keys = {"max_prompt_length"} |
| 1079 | + expected_direct_hp_keys = {"use_kl_loss", "kl_loss_coef"} |
| 1080 | + all_user_keys = expected_override_keys | expected_recipe_keys | expected_direct_hp_keys |
| 1081 | + |
| 1082 | + # All user-provided keys must be present in the user-override result |
| 1083 | + for key in all_user_keys: |
| 1084 | + assert key in result_hp, ( |
| 1085 | + f"User-provided key '{key}' missing from serverless (compute=None) result" |
| 1086 | + ) |
| 1087 | + |
| 1088 | + # Dynamically compute recipe keys NOT in the override spec by fetching |
| 1089 | + # the full resolved recipe and subtracting the spec keys + user-provided keys. |
| 1090 | + # Only consider keys with non-None values (None keys are skipped by _apply_recipe). |
| 1091 | + resolved_recipe = rlvr_trainer.get_resolved_recipe() |
| 1092 | + flat_recipe = flatten_resolved_recipe(resolved_recipe) |
| 1093 | + all_recipe_keys = {k for k, v in flat_recipe.items() if v is not None} |
| 1094 | + override_spec_keys = set(rlvr_trainer.hyperparameters._specs.keys()) |
| 1095 | + recipe_internal_keys_not_in_spec = all_recipe_keys - override_spec_keys - all_user_keys |
| 1096 | + |
| 1097 | + logger.info(f"All recipe keys from Hub ({len(all_recipe_keys)}): {sorted(all_recipe_keys)}") |
| 1098 | + logger.info(f"Override spec keys ({len(override_spec_keys)}): {sorted(override_spec_keys)}") |
| 1099 | + logger.info( |
| 1100 | + f"Recipe internal keys NOT in spec ({len(recipe_internal_keys_not_in_spec)}): " |
| 1101 | + f"{sorted(recipe_internal_keys_not_in_spec)}" |
| 1102 | + ) |
| 1103 | + |
| 1104 | + assert len(recipe_internal_keys_not_in_spec) > 0, ( |
| 1105 | + "Expected recipe template to have keys beyond the override spec, but found none." |
| 1106 | + ) |
| 1107 | + |
| 1108 | + # These internal recipe keys must NOT appear in the serverless result |
| 1109 | + leaked_keys = recipe_internal_keys_not_in_spec & set(result_hp.keys()) |
| 1110 | + assert not leaked_keys, ( |
| 1111 | + f"Non-overridable recipe template keys leaked into the serverless hyperparameters: " |
| 1112 | + f"{sorted(leaked_keys)}. These keys are not in the override spec and should be " |
| 1113 | + f"excluded when compute=None (serverless)." |
| 1114 | + ) |
| 1115 | + |
| 1116 | + # The full recipe (compute set) MUST contain these internal keys |
| 1117 | + missing_from_full = recipe_internal_keys_not_in_spec - set(full_hp.keys()) |
| 1118 | + assert not missing_from_full, ( |
| 1119 | + f"Full recipe (compute set) is missing expected internal keys: " |
| 1120 | + f"{sorted(missing_from_full)}. The full recipe path should include all template keys." |
| 1121 | + ) |
| 1122 | + |
| 1123 | + # The user-override result should not exceed the 100-key CreateTrainingJob limit |
| 1124 | + assert len(result_hp) <= 100, ( |
| 1125 | + f"Serverless hyperparameters have {len(result_hp)} keys, exceeding the " |
| 1126 | + f"CreateTrainingJob limit of 100. Non-overridden recipe keys are leaking through." |
| 1127 | + ) |
| 1128 | + |
| 1129 | + finally: |
| 1130 | + os.unlink(recipe_path) |
| 1131 | + |
| 1132 | + def test_sft_nova_serverless_only_user_override_keys_applied(self, sagemaker_session_us_east_1): |
| 1133 | + """Test that Nova SFT serverless path (compute=None) excludes non-overridden recipe keys. |
| 1134 | +
|
| 1135 | + Same principle as the RLVR Nemotron test: when compute is None, |
| 1136 | + internal recipe template keys that the user never touched must not appear |
| 1137 | + in the hyperparameters sent to CreateTrainingJob. |
| 1138 | + """ |
| 1139 | + sft_trainer = SFTTrainer( |
| 1140 | + model="nova-textgeneration-lite-v2", |
| 1141 | + training_type=TrainingType.LORA, |
| 1142 | + model_package_group="sdk-test-finetuned-models", |
| 1143 | + training_dataset="s3://sagemaker-us-east-1-784379639078/input_data/sft-nova/sft_200_samples.jsonl", |
| 1144 | + s3_output_path="s3://sagemaker-us-east-1-784379639078/output/", |
| 1145 | + sagemaker_session=sagemaker_session_us_east_1, |
| 1146 | + accept_eula=True, |
| 1147 | + base_job_name="sft-nova-override-keys-integ", |
| 1148 | + overrides={ |
| 1149 | + "training_config": { |
| 1150 | + "learning_rate": 3e-5, |
| 1151 | + "max_steps": 50, |
| 1152 | + } |
| 1153 | + }, |
| 1154 | + ) |
| 1155 | + |
| 1156 | + sft_trainer.hyperparameters.warmup_steps = 5 |
| 1157 | + sft_trainer.hyperparameters.weight_decay = 0.01 |
| 1158 | + |
| 1159 | + baseline_hp = sft_trainer.hyperparameters.to_dict() |
| 1160 | + |
| 1161 | + # Serverless path (compute=None): only user-provided keys applied |
| 1162 | + assert sft_trainer.compute is None |
| 1163 | + result_hp = sft_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy()) |
| 1164 | + |
| 1165 | + # Simulate serverful path (compute set): full recipe applied |
| 1166 | + sft_trainer.compute = TrainingJobCompute(instance_type="ml.p5.48xlarge", instance_count=1) |
| 1167 | + full_hp = sft_trainer._apply_recipe_to_hyperparameters(baseline_hp.copy()) |
| 1168 | + sft_trainer.compute = None # reset |
| 1169 | + |
| 1170 | + logger.info(f"Nova SFT — Baseline HP keys: {len(baseline_hp)}") |
| 1171 | + logger.info(f"Nova SFT — User-override-only HP keys: {len(result_hp)}") |
| 1172 | + logger.info(f"Nova SFT — Full recipe HP keys: {len(full_hp)}") |
| 1173 | + |
| 1174 | + # Keys the user explicitly provided |
| 1175 | + expected_override_keys = {"learning_rate", "max_steps"} |
| 1176 | + expected_direct_hp_keys = {"warmup_steps", "weight_decay"} |
| 1177 | + all_user_keys = expected_override_keys | expected_direct_hp_keys |
| 1178 | + |
| 1179 | + for key in all_user_keys: |
| 1180 | + assert key in result_hp, ( |
| 1181 | + f"User-provided key '{key}' missing from serverless (compute=None) result" |
| 1182 | + ) |
| 1183 | + |
| 1184 | + # Full recipe should have more keys than the user-override-only result |
| 1185 | + full_only_keys = set(full_hp.keys()) - set(result_hp.keys()) |
| 1186 | + assert len(full_only_keys) > 0, ( |
| 1187 | + "Full recipe should contain additional keys beyond the user-override-only result." |
| 1188 | + ) |
| 1189 | + logger.info( |
| 1190 | + f"Nova SFT — Keys excluded from serverless path: " |
| 1191 | + f"{len(full_only_keys)} keys — {sorted(list(full_only_keys))}" |
| 1192 | + ) |
| 1193 | + |
| 1194 | + # Dynamically compute recipe keys NOT in the override spec by fetching |
| 1195 | + # the full resolved recipe and subtracting the spec keys + user-provided keys. |
| 1196 | + # Only consider keys with non-None values (None keys are skipped by _apply_recipe). |
| 1197 | + resolved_recipe = sft_trainer.get_resolved_recipe() |
| 1198 | + flat_recipe = flatten_resolved_recipe(resolved_recipe) |
| 1199 | + all_recipe_keys = {k for k, v in flat_recipe.items() if v is not None} |
| 1200 | + override_spec_keys = set(sft_trainer.hyperparameters._specs.keys()) |
| 1201 | + recipe_internal_keys_not_in_spec = all_recipe_keys - override_spec_keys - all_user_keys |
| 1202 | + |
| 1203 | + logger.info(f"Nova SFT — All recipe keys from Hub ({len(all_recipe_keys)}): {sorted(all_recipe_keys)}") |
| 1204 | + logger.info(f"Nova SFT — Override spec keys ({len(override_spec_keys)}): {sorted(override_spec_keys)}") |
| 1205 | + logger.info( |
| 1206 | + f"Nova SFT — Recipe internal keys NOT in spec ({len(recipe_internal_keys_not_in_spec)}): " |
| 1207 | + f"{sorted(recipe_internal_keys_not_in_spec)}" |
| 1208 | + ) |
| 1209 | + |
| 1210 | + assert len(recipe_internal_keys_not_in_spec) > 0, ( |
| 1211 | + "Expected Nova recipe template to have keys beyond the override spec, but found none." |
| 1212 | + ) |
| 1213 | + |
| 1214 | + # These internal recipe keys must NOT appear in the serverless result |
| 1215 | + leaked_keys = recipe_internal_keys_not_in_spec & set(result_hp.keys()) |
| 1216 | + assert not leaked_keys, ( |
| 1217 | + f"Non-overridable Nova recipe template keys leaked into serverless hyperparameters: " |
| 1218 | + f"{sorted(leaked_keys)}. These keys are not in the override spec and should be " |
| 1219 | + f"excluded when compute=None (serverless)." |
| 1220 | + ) |
| 1221 | + |
| 1222 | + # The full recipe (compute set) MUST contain these internal keys |
| 1223 | + missing_from_full = recipe_internal_keys_not_in_spec - set(full_hp.keys()) |
| 1224 | + assert not missing_from_full, ( |
| 1225 | + f"Full recipe (compute set) is missing expected internal keys: " |
| 1226 | + f"{sorted(missing_from_full)}. The full recipe path should include all template keys." |
| 1227 | + ) |
| 1228 | + |
| 1229 | + assert len(result_hp) <= 100, ( |
| 1230 | + f"Serverless hyperparameters have {len(result_hp)} keys, exceeding the " |
| 1231 | + f"CreateTrainingJob limit of 100." |
| 1232 | + ) |
0 commit comments