Skip to content

Commit fd79cf5

Browse files
committed
fix(tuner): Upload sourcedir.tar.gz for framework container compatibility
The HPT API uses the legacy framework container path which expects sagemaker_submit_directory (a tar.gz on S3) to be downloaded and extracted to /opt/ml/code/. The previous approach of using a 'code' input channel mounted the code at /opt/ml/input/data/code/ instead, causing 'No such file or directory' errors. Fix: Create and upload sourcedir.tar.gz to S3, set both sagemaker_program and sagemaker_submit_directory hyperparameters. Remove the separate 'code' input channel since the framework container handles code extraction via sagemaker_submit_directory.
1 parent 3dee1f2 commit fd79cf5

1 file changed

Lines changed: 40 additions & 14 deletions

File tree

  • sagemaker-train/src/sagemaker/train

‎sagemaker-train/src/sagemaker/train/tuner.py‎

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -533,22 +533,48 @@ def _build_driver_and_code_channels(cls, model_trainer):
533533
ignore_patterns=source_code.ignore_patterns,
534534
)
535535

536-
# Upload code channel if source_dir is set
537-
code_channel = None
538-
if source_code.source_dir:
539-
code_channel = model_trainer.create_input_data_channel(
540-
channel_name=SM_CODE,
541-
data_source=source_code.source_dir,
542-
key_prefix=key_prefix,
543-
ignore_patterns=source_code.ignore_patterns,
544-
)
545-
546536
# Store channels on model_trainer so _build_training_job_definition can pick them up
547-
if not hasattr(model_trainer, "_tuner_channels"):
548-
model_trainer._tuner_channels = []
549537
model_trainer._tuner_channels = [sm_drivers_channel]
550-
if code_channel:
551-
model_trainer._tuner_channels.append(code_channel)
538+
539+
# Set script mode hyperparameters required by framework containers.
540+
# The framework container (PyTorch, TF) uses sagemaker_program to find the entry script
541+
# and sagemaker_submit_directory to download source code to /opt/ml/code/.
542+
if model_trainer.hyperparameters is None:
543+
model_trainer.hyperparameters = {}
544+
model_trainer.hyperparameters["sagemaker_program"] = source_code.entry_script
545+
546+
# Upload sourcedir.tar.gz for the legacy framework container path.
547+
# The HPT API doesn't support container_entrypoint, so the framework container
548+
# uses sagemaker_submit_directory to download and extract code to /opt/ml/code/.
549+
if source_code.source_dir and not _is_valid_s3_uri(source_code.source_dir):
550+
import tarfile
551+
import tempfile
552+
553+
session = model_trainer.sagemaker_session
554+
bucket = session.default_bucket()
555+
s3_key = f"{key_prefix}/sourcedir/sourcedir.tar.gz"
556+
557+
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
558+
tar_path = tmp.name
559+
try:
560+
with tarfile.open(tar_path, "w:gz") as tar:
561+
for root, _dirs, files in os.walk(source_code.source_dir):
562+
for f in files:
563+
fpath = os.path.join(root, f)
564+
arcname = os.path.relpath(fpath, source_code.source_dir)
565+
tar.add(fpath, arcname=arcname)
566+
s3_client = session.boto_session.client(
567+
"s3", region_name=session.boto_region_name
568+
)
569+
s3_client.upload_file(tar_path, bucket, s3_key)
570+
model_trainer.hyperparameters["sagemaker_submit_directory"] = (
571+
f"s3://{bucket}/{s3_key}"
572+
)
573+
finally:
574+
if os.path.exists(tar_path):
575+
os.remove(tar_path)
576+
elif source_code.source_dir and _is_valid_s3_uri(source_code.source_dir):
577+
model_trainer.hyperparameters["sagemaker_submit_directory"] = source_code.source_dir
552578

553579
# Store the temp dir reference to prevent cleanup
554580
model_trainer._tuner_temp_dir = temp_dir

0 commit comments

Comments
 (0)