Skip to content

Commit f83dea4

Browse files
committed
add local dependencies to source_dir
1 parent 70dc107 commit f83dea4

3 files changed

Lines changed: 69 additions & 2 deletions

File tree

‎sagemaker-core/src/sagemaker/core/processing.py‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,6 +1156,7 @@ def _package_code(
11561156
self,
11571157
entry_point,
11581158
source_dir,
1159+
dependencies,
11591160
requirements,
11601161
job_name,
11611162
kms_key,
@@ -1165,13 +1166,27 @@ def _package_code(
11651166
If source_dir is an S3 URI, it is used directly as the code payload
11661167
(no local packaging or upload is performed). The S3 URI should point to
11671168
either a tar.gz archive or an S3 prefix containing the source code.
1169+
1170+
Args:
1171+
entry_point (str): Path to the entry point script.
1172+
source_dir (str): Local directory, S3 URI, or None.
1173+
dependencies (list[str]): Additional local directories to include
1174+
in the tar.gz bundle (default: None). Not supported with S3 source_dir.
1175+
requirements (str): Path to requirements.txt relative to source_dir.
1176+
job_name (str): Processing job name (used in S3 key).
1177+
kms_key (str): KMS key for S3 upload encryption.
11681178
"""
11691179
import tarfile
11701180
import tempfile
11711181

11721182
# S3 source_dir: use it directly without local packaging.
11731183
# This restores v2 behavior where S3 paths with tar.gz archives were supported.
11741184
if self._is_s3_uri(source_dir):
1185+
if dependencies:
1186+
raise ValueError(
1187+
"dependencies is not supported when source_dir is an S3 URI. "
1188+
"Bundle dependencies into the S3 tar.gz archive instead."
1189+
)
11751190
return self._resolve_s3_source_dir(source_dir)
11761191

11771192
# If source_dir is not provided, use the directory containing entry_point
@@ -1188,13 +1203,22 @@ def _package_code(
11881203
if not os.path.exists(source_dir):
11891204
raise ValueError(f"source_dir does not exist: {source_dir}")
11901205

1191-
# Create tar.gz with source_dir contents
1206+
# Create tar.gz with source_dir contents + dependencies
11921207
with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
11931208
with tarfile.open(tmp.name, "w:gz") as tar:
1209+
# Add all files from source_dir
11941210
for item in os.listdir(source_dir):
11951211
item_path = os.path.join(source_dir, item)
11961212
tar.add(item_path, arcname=item)
11971213

1214+
# Add dependency directories at the root of the archive
1215+
for dep_path in (dependencies or []):
1216+
if not os.path.isabs(dep_path):
1217+
dep_path = os.path.abspath(dep_path)
1218+
if not os.path.exists(dep_path):
1219+
raise ValueError(f"Dependency path does not exist: {dep_path}")
1220+
tar.add(dep_path, arcname=os.path.basename(dep_path))
1221+
11981222
s3_uri = s3.s3_path_join(
11991223
self._s3_code_prefix(),
12001224
job_name,
@@ -1218,6 +1242,7 @@ def run(
12181242
self,
12191243
code: str,
12201244
source_dir: Optional[str] = None,
1245+
dependencies: Optional[List[str]] = None,
12211246
requirements: Optional[str] = None,
12221247
inputs: Optional[List[ProcessingInput]] = None,
12231248
outputs: Optional[List["ProcessingOutput"]] = None,
@@ -1236,7 +1261,13 @@ def run(
12361261
framework script to run.
12371262
source_dir (str): Path (absolute, relative or an S3 URI) to a directory
12381263
with any other processing source code dependencies aside from the entry
1239-
point file (default: None).
1264+
point file (default: None). If ``source_dir`` is an S3 URI, it must
1265+
point to a tar.gz file named ``sourcedir.tar.gz``.
1266+
dependencies (list[str]): A list of paths to directories (absolute or
1267+
relative) with any additional libraries that will be exported to the
1268+
container (default: None). The library folders are copied into the
1269+
same tar.gz bundle as source_dir. Not supported when source_dir is
1270+
an S3 URI.
12401271
requirements (str): Path to a requirements.txt file relative to source_dir
12411272
(default: None).
12421273
inputs (list[:class:`~sagemaker.processing.ProcessingInput`]): Input files for
@@ -1265,6 +1296,7 @@ def run(
12651296
s3_runproc_sh, inputs, job_name = self._pack_and_upload_code(
12661297
code,
12671298
source_dir,
1299+
dependencies,
12681300
requirements,
12691301
job_name,
12701302
inputs,
@@ -1289,6 +1321,7 @@ def _pack_and_upload_code(
12891321
self,
12901322
code,
12911323
source_dir,
1324+
dependencies,
12921325
requirements,
12931326
job_name,
12941327
inputs,
@@ -1306,6 +1339,7 @@ def _pack_and_upload_code(
13061339
s3_payload = self._package_code(
13071340
entry_point=code,
13081341
source_dir=source_dir,
1342+
dependencies=dependencies,
13091343
requirements=requirements,
13101344
job_name=job_name,
13111345
kms_key=kms_key,
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
"""Helper module to verify cross-file imports work from S3 source_dir."""
2+
3+
4+
def get_greeting(name: str) -> str:
5+
"""Return a greeting string."""
6+
return f"Hello from S3 source_dir, {name}!"
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
"""Simple processing script for S3 source_dir integ test.
2+
3+
This script validates that:
4+
1. It can be executed from an S3-based source_dir
5+
2. It can import from a sibling module in the same source bundle
6+
"""
7+
import os
8+
import json
9+
10+
from helpers import get_greeting
11+
12+
13+
if __name__ == "__main__":
14+
output_dir = "/opt/ml/processing/output"
15+
os.makedirs(output_dir, exist_ok=True)
16+
17+
result = {
18+
"status": "success",
19+
"greeting": get_greeting("integration-test"),
20+
"source_dir_type": "s3",
21+
}
22+
23+
output_path = os.path.join(output_dir, "result.json")
24+
with open(output_path, "w") as f:
25+
json.dump(result, f)
26+
27+
print(f"Processing complete. Output written to {output_path}")

0 commit comments

Comments
 (0)