Skip to content

Commit 50fe39c

Browse files
fix: apply ModelTrainer output KMS key to tuner source upload (#6555) (#6559)
HyperparameterTuner._build_driver_and_code_channels() uploaded the local sourcedir.tar.gz with s3_client.upload_file() and no ExtraArgs, so the ModelTrainer's output_data_config.kms_key_id was never applied. The staged code could be encrypted with the wrong key, and uploads fail where a bucket policy requires explicit SSE-KMS headers. Build ExtraArgs via ModelTrainer._resolve_upload_extra_args() and forward them to upload_file (parity with create_input_data_channel); the result is None when no key is configured, so default behavior is unchanged. Add regression tests covering the KMS and no-KMS cases. Co-authored-by: Mufaddal Rohawala <mufi@amazon.com>
1 parent e6f2dac commit 50fe39c

2 files changed

Lines changed: 43 additions & 1 deletion

File tree

‎sagemaker-train/src/sagemaker/train/tuner.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -590,7 +590,10 @@ def _build_driver_and_code_channels(cls, model_trainer):
590590
arcname = os.path.relpath(fpath, source_code.source_dir)
591591
tar.add(fpath, arcname=arcname)
592592
s3_client = session.boto_session.client("s3", region_name=session.boto_region_name)
593-
s3_client.upload_file(tar_path, bucket, s3_key)
593+
# Encrypt staged source with the trainer's output KMS key (parity with
594+
# create_input_data_channel); None when no key is set. See issue #6555.
595+
extra_args = model_trainer._resolve_upload_extra_args()
596+
s3_client.upload_file(tar_path, bucket, s3_key, ExtraArgs=extra_args)
594597
model_trainer.hyperparameters["sagemaker_submit_directory"] = (
595598
f"s3://{bucket}/{s3_key}"
596599
)

‎sagemaker-train/tests/unit/train/test_tuner_driver_channels.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,45 @@ def test_uploads_sourcedir_tar_gz(self, tmp_path):
277277
# sagemaker_submit_directory set
278278
assert trainer.hyperparameters["sagemaker_submit_directory"].startswith("s3://my-bucket/")
279279

280+
def _upload_trainer(self, tmp_path, extra_args):
281+
"""Mock trainer whose source upload returns ``extra_args`` from the resolver."""
282+
src_dir = str(tmp_path / "src")
283+
os.makedirs(src_dir)
284+
with open(os.path.join(src_dir, "train.py"), "w") as f:
285+
f.write("print('hello')")
286+
287+
mock_s3_client = MagicMock()
288+
mock_session = MagicMock()
289+
mock_session.default_bucket.return_value = "my-bucket"
290+
mock_session.default_bucket_prefix = None
291+
mock_session.boto_session.client.return_value = mock_s3_client
292+
mock_session.boto_region_name = "us-west-2"
293+
294+
trainer = _mock_model_trainer(source_code=_make_source_code(source_dir=src_dir))
295+
trainer.sagemaker_session = mock_session
296+
trainer.create_input_data_channel.return_value = _create_channel("sm_drivers")
297+
trainer.hyperparameters = {}
298+
trainer._resolve_upload_extra_args.return_value = extra_args
299+
return trainer, mock_s3_client
300+
301+
def test_uploads_sourcedir_with_output_kms_key(self, tmp_path):
302+
"""Local source upload must forward the trainer's KMS ExtraArgs (issue #6555)."""
303+
extra_args = {"ServerSideEncryption": "aws:kms", "SSEKMSKeyId": "my-kms-key"}
304+
trainer, mock_s3_client = self._upload_trainer(tmp_path, extra_args)
305+
306+
HyperparameterTuner._build_driver_and_code_channels(trainer)
307+
308+
trainer._resolve_upload_extra_args.assert_called_once_with()
309+
assert mock_s3_client.upload_file.call_args.kwargs["ExtraArgs"] == extra_args
310+
311+
def test_uploads_sourcedir_without_kms_key(self, tmp_path):
312+
"""No KMS key resolved means ExtraArgs=None, preserving default behavior."""
313+
trainer, mock_s3_client = self._upload_trainer(tmp_path, None)
314+
315+
HyperparameterTuner._build_driver_and_code_channels(trainer)
316+
317+
assert mock_s3_client.upload_file.call_args.kwargs["ExtraArgs"] is None
318+
280319
def test_sets_submit_directory_for_s3_source_dir(self, tmp_path):
281320
"""Should set sagemaker_submit_directory directly when source_dir is an S3 URI."""
282321
source_code = _make_source_code(source_dir="s3://my-bucket/code/sourcedir.tar.gz")

0 commit comments

Comments
 (0)