Skip to content

Commit e28bfc4

Browse files
author
Mohamed Zeidan
committed
fix: make dependencies optional in FrameworkProcessor code-packaging helpers
_package_code and _pack_and_upload_code declared dependencies as a required positional arg, but the unit tests (and any direct callers) invoke them without it, causing: TypeError: _package_code() missing 1 required positional argument: 'dependencies' Move dependencies to the end of both signatures with a None default so it is optional, matching the public run() signature. Also apply black formatting and remove unused imports/vars in test_processing.py.
1 parent 46f4f71 commit e28bfc4

2 files changed

Lines changed: 35 additions & 28 deletions

File tree

‎sagemaker-core/src/sagemaker/core/processing.py‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,10 @@ def _normalize_outputs(self, outputs=None):
489489
# If the output's s3_uri is not an s3_uri, create one.
490490
parse_result = urlparse(output.s3_output.s3_uri)
491491
if parse_result.scheme != "s3":
492-
if getattr(self.sagemaker_session, "local_mode", False) and parse_result.scheme == "file":
492+
if (
493+
getattr(self.sagemaker_session, "local_mode", False)
494+
and parse_result.scheme == "file"
495+
):
493496
normalized_outputs.append(output)
494497
continue
495498
if _pipeline_config:
@@ -1156,10 +1159,10 @@ def _package_code(
11561159
self,
11571160
entry_point,
11581161
source_dir,
1159-
dependencies,
11601162
requirements,
11611163
job_name,
11621164
kms_key,
1165+
dependencies=None,
11631166
):
11641167
"""Package and upload code to S3.
11651168
@@ -1212,7 +1215,7 @@ def _package_code(
12121215
tar.add(item_path, arcname=item)
12131216

12141217
# Add dependency directories at the root of the archive
1215-
for dep_path in (dependencies or []):
1218+
for dep_path in dependencies or []:
12161219
if not os.path.isabs(dep_path):
12171220
dep_path = os.path.abspath(dep_path)
12181221
if not os.path.exists(dep_path):
@@ -1252,7 +1255,7 @@ def run(
12521255
job_name: Optional[str] = None,
12531256
experiment_config: Optional[Dict[str, str]] = None,
12541257
kms_key: Optional[str] = None,
1255-
entry_point: Optional[str] = None
1258+
entry_point: Optional[str] = None,
12561259
):
12571260
"""Runs a processing job.
12581261
@@ -1296,12 +1299,12 @@ def run(
12961299
s3_runproc_sh, inputs, job_name = self._pack_and_upload_code(
12971300
code,
12981301
source_dir,
1299-
dependencies,
13001302
requirements,
13011303
job_name,
13021304
inputs,
13031305
kms_key,
1304-
entry_point
1306+
entry_point,
1307+
dependencies=dependencies,
13051308
)
13061309

13071310
# Submit a processing job.
@@ -1321,12 +1324,12 @@ def _pack_and_upload_code(
13211324
self,
13221325
code,
13231326
source_dir,
1324-
dependencies,
13251327
requirements,
13261328
job_name,
13271329
inputs,
13281330
kms_key=None,
1329-
entry_point=None
1331+
entry_point=None,
1332+
dependencies=None,
13301333
):
13311334
"""Pack local code bundle and upload to Amazon S3."""
13321335
if code.startswith("s3://"):
@@ -1409,7 +1412,9 @@ def _set_entrypoint(self, command, user_script_name):
14091412
)
14101413
self.entrypoint = self.framework_entrypoint_command + [user_script_location]
14111414

1412-
def _create_and_upload_runproc(self, user_script, kms_key, entrypoint_s3_uri, entry_point=None, source_dir=None):
1415+
def _create_and_upload_runproc(
1416+
self, user_script, kms_key, entrypoint_s3_uri, entry_point=None, source_dir=None
1417+
):
14131418
"""Create runproc shell script and upload to S3 bucket."""
14141419
from sagemaker.core.workflow.utilities import _pipeline_config, hash_object
14151420

@@ -1439,7 +1444,9 @@ def _create_and_upload_runproc(self, user_script, kms_key, entrypoint_s3_uri, en
14391444

14401445
return s3_runproc_sh
14411446

1442-
def _generate_framework_script(self, user_script: str, entry_point: str = None, source_dir: str = None) -> str:
1447+
def _generate_framework_script(
1448+
self, user_script: str, entry_point: str = None, source_dir: str = None
1449+
) -> str:
14431450
"""Generate the framework entrypoint file (as text) for a processing job."""
14441451
if entry_point:
14451452
return self._generate_custom_framework_script(user_script, entry_point, source_dir)
@@ -1548,11 +1555,13 @@ def _generate_custom_framework_script(
15481555
entry_point_content = f.read()
15491556

15501557
# Generate the script with embedded entry_point content
1551-
return dedent("""\
1558+
return dedent(
1559+
"""\
15521560
{entry_point_content}
15531561
15541562
{entry_point_command} {entry_point} "$@"
1555-
""").format(
1563+
"""
1564+
).format(
15561565
entry_point_content=entry_point_content,
15571566
entry_point_command=" ".join(self.command),
15581567
entry_point=user_script,

‎sagemaker-core/tests/unit/test_processing.py‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import pytest
1515
import os
1616
import tempfile
17-
from unittest.mock import Mock, patch, MagicMock, mock_open
17+
from unittest.mock import Mock, patch
1818
from sagemaker.core.processing import (
1919
Processor,
2020
ScriptProcessor,
@@ -48,9 +48,6 @@ def mock_session():
4848

4949
class TestProcessorNormalizeArgs:
5050
def test_normalize_args_with_pipeline_variable_code(self, mock_session):
51-
from sagemaker.core.workflow.pipeline_context import PipelineSession
52-
from sagemaker.core.workflow import is_pipeline_variable
53-
5451
processor = Processor(
5552
role="arn:aws:iam::123456789012:role/SageMakerRole",
5653
image_uri="test-image:latest",
@@ -238,8 +235,6 @@ def test_normalize_outputs_invalid_type(self, mock_session):
238235
processor._normalize_outputs(["invalid"])
239236

240237

241-
242-
243238
class TestBugConditionFileUriReplacedInLocalMode:
244239
"""Bug condition exploration test: file:// URIs should be preserved in local mode.
245240
@@ -444,7 +439,9 @@ def test_pipeline_variable_uri_skips_normalization(self, session_local_mode_fals
444439
[{"key": "value"}],
445440
],
446441
)
447-
def test_non_processing_output_raises_type_error(self, invalid_output, session_local_mode_false):
442+
def test_non_processing_output_raises_type_error(
443+
self, invalid_output, session_local_mode_false
444+
):
448445
"""Non-ProcessingOutput objects must raise TypeError.
449446
450447
**Validates: Requirements 3.4**
@@ -882,9 +879,7 @@ def test_generate_framework_script_with_custom_entry_point(self, mock_session):
882879
entry_point_path = f.name
883880

884881
try:
885-
script = processor._generate_framework_script(
886-
"train.py", entry_point=entry_point_path
887-
)
882+
script = processor._generate_framework_script("train.py", entry_point=entry_point_path)
888883
assert custom_script_content in script
889884
assert "python3 train.py" in script
890885
assert "tar -xzf sourcedir.tar.gz" not in script
@@ -1042,7 +1037,7 @@ def test_start_new_submit_success(self, mock_session):
10421037
},
10431038
):
10441039
with patch("sagemaker.core.processing.serialize", return_value={}):
1045-
with patch("sagemaker.core.processing.ProcessingJob") as mock_job:
1040+
with patch("sagemaker.core.processing.ProcessingJob"):
10461041
with patch(
10471042
"sagemaker.core.utils.code_injection.codec.transform",
10481043
return_value={"processing_job_name": "test-job"},
@@ -1220,7 +1215,6 @@ def test_package_code_source_dir_not_exists(self, mock_session):
12201215
kms_key=None,
12211216
)
12221217

1223-
12241218
def test_package_code_with_code_location(self, mock_session):
12251219
processor = FrameworkProcessor(
12261220
role="arn:aws:iam::123456789012:role/SageMakerRole",
@@ -1477,8 +1471,6 @@ def test_generate_job_name_with_invalid_chars(self, mock_session):
14771471

14781472
class TestProcessorWithPipelineVariable:
14791473
def test_get_process_args_with_pipeline_variable_role(self, mock_session):
1480-
from sagemaker.core.workflow import is_pipeline_variable
1481-
14821474
role_var = Mock()
14831475

14841476
with patch("sagemaker.core.processing.is_pipeline_variable", return_value=True):
@@ -1524,11 +1516,17 @@ def test_start_new_removes_tags_from_processing_job(self, mock_session):
15241516
"tags": [{"Key": "Key", "Value": "Value"}],
15251517
},
15261518
):
1527-
with patch("sagemaker.core.processing.serialize", return_value={"tags": [{"Key": "Key", "Value": "Value"}]}):
1519+
with patch(
1520+
"sagemaker.core.processing.serialize",
1521+
return_value={"tags": [{"Key": "Key", "Value": "Value"}]},
1522+
):
15281523
with patch("sagemaker.core.processing.ProcessingJob") as mock_job_class:
15291524
with patch(
15301525
"sagemaker.core.utils.code_injection.codec.transform",
1531-
return_value={"processing_job_name": "test-job", "tags": [{"Key": "Key", "Value": "Value"}]},
1526+
return_value={
1527+
"processing_job_name": "test-job",
1528+
"tags": [{"Key": "Key", "Value": "Value"}],
1529+
},
15321530
):
15331531
processor._start_new([], [], None)
15341532
# Verify ProcessingJob was called without tags

0 commit comments

Comments
 (0)