Skip to content

Commit 7921169

Browse files
committed
fix(processing): Fix V3 shape regressions in processors
PySparkProcessor still built ProcessingInput/ProcessingOutput with the V2 source/destination keyword signature, so spark_event_logs_s3_uri and the configuration channel failed on V3. Build them with the V3 shapes (ProcessingS3Input/ProcessingS3Output) instead. Validate that submit_py_files, submit_jars and submit_files are lists and raise a clear ValueError otherwise. FrameworkProcessor.run(requirements=...) now threads the file name into the generated runproc.sh instead of hardcoding requirements.txt. Processor.run defaults kms_key to the configured output_kms_key so uploaded code is encrypted with the same key as job outputs. Fixes #3809 Fixes #6253 Fixes #6252 Fixes #5805 Fixes #4874 --- X-AI-Prompt: Fix S-effort PySDK V3 bugs, processing-spark theme X-AI-Tool: Kiro
1 parent 7ac2708 commit 7921169

4 files changed

Lines changed: 249 additions & 16 deletions

File tree

‎sagemaker-core/src/sagemaker/core/processing.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,8 @@ def run(
332332
is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession`.
333333
However, the value of `TrialComponentDisplayName` is honored for display in Studio.
334334
kms_key (str): The ARN of the KMS key that is used to encrypt the
335-
user code file (default: None).
335+
user code file. If not provided, the processor's configured
336+
``output_kms_key`` is used (default: None).
336337
Returns:
337338
None or pipeline step arguments in case the Processor instance is built with
338339
:class:`~sagemaker.workflow.pipeline_context.PipelineSession`
@@ -343,6 +344,12 @@ def run(
343344
raise ValueError("""Logs can only be shown if wait is set to True.
344345
Please either set wait to True or set logs to False.""")
345346

347+
# When no explicit code-encryption key is given, fall back to the
348+
# configured output KMS key so the uploaded code and job outputs are
349+
# encrypted with the same key.
350+
if kms_key is None:
351+
kms_key = self.output_kms_key
352+
346353
normalized_inputs, normalized_outputs = self._normalize_args(
347354
job_name=job_name,
348355
arguments=arguments,
@@ -903,7 +910,8 @@ def run(
903910
is built with :class:`~sagemaker.workflow.pipeline_context.PipelineSession`.
904911
However, the value of `TrialComponentDisplayName` is honored for display in Studio.
905912
kms_key (str): The ARN of the KMS key that is used to encrypt the
906-
user code file (default: None).
913+
user code file. If not provided, the processor's configured
914+
``output_kms_key`` is used (default: None).
907915
Returns:
908916
None or pipeline step arguments in case the Processor instance is built with
909917
:class:`~sagemaker.workflow.pipeline_context.PipelineSession`
@@ -914,7 +922,7 @@ def run(
914922
inputs=inputs,
915923
outputs=outputs,
916924
code=code,
917-
kms_key=kms_key,
925+
kms_key=kms_key if kms_key is not None else self.output_kms_key,
918926
)
919927

920928
experiment_config = check_and_get_run_experiment_config(experiment_config)
@@ -1481,6 +1489,7 @@ def _pack_and_upload_code(
14811489
entry_point,
14821490
source_dir,
14831491
install_requirements_dir,
1492+
requirements=requirements,
14841493
)
14851494

14861495
return s3_runproc_sh, inputs, job_name
@@ -1526,13 +1535,14 @@ def _create_and_upload_runproc(
15261535
entry_point=None,
15271536
source_dir=None,
15281537
install_requirements_dir=None,
1538+
requirements=None,
15291539
):
15301540
"""Create runproc shell script and upload to S3 bucket."""
15311541
from sagemaker.core.workflow.utilities import _pipeline_config, hash_object
15321542

15331543
if _pipeline_config and _pipeline_config.pipeline_name:
15341544
runproc_file_str = self._generate_framework_script(
1535-
user_script, entry_point, source_dir, install_requirements_dir
1545+
user_script, entry_point, source_dir, install_requirements_dir, requirements
15361546
)
15371547
runproc_file_hash = hash_object(runproc_file_str)
15381548
s3_uri = s3.s3_path_join(
@@ -1551,7 +1561,7 @@ def _create_and_upload_runproc(
15511561
else:
15521562
s3_runproc_sh = s3.S3Uploader.upload_string_as_file_body(
15531563
self._generate_framework_script(
1554-
user_script, entry_point, source_dir, install_requirements_dir
1564+
user_script, entry_point, source_dir, install_requirements_dir, requirements
15551565
),
15561566
desired_s3_uri=entrypoint_s3_uri,
15571567
kms_key=kms_key,
@@ -1566,6 +1576,7 @@ def _generate_framework_script(
15661576
entry_point: str = None,
15671577
source_dir: str = None,
15681578
install_requirements_dir: str = None,
1579+
requirements: str = None,
15691580
) -> str:
15701581
"""Generate the framework entrypoint file (as text) for a processing job."""
15711582
if entry_point:
@@ -1574,6 +1585,7 @@ def _generate_framework_script(
15741585
)
15751586

15761587
install_requirements_dir = install_requirements_dir or self._SOURCE_CODE_CONTAINER_DIR
1588+
requirements_file = os.path.basename(requirements) if requirements else "requirements.txt"
15771589

15781590
return dedent("""\
15791591
#!/bin/bash
@@ -1597,16 +1609,17 @@ def _generate_framework_script(
15971609
exit 1
15981610
fi
15991611
1600-
if [[ -f 'requirements.txt' ]]; then
1612+
if [[ -f '{requirements_file}' ]]; then
16011613
# Some py3 containers has typing, which may breaks pip install
16021614
pip uninstall --yes typing
16031615
1604-
python3 {install_requirements_dir}/install_requirements.py requirements.txt
1616+
python3 {install_requirements_dir}/install_requirements.py {requirements_file}
16051617
fi
16061618
16071619
{entry_point_command} {entry_point} "$@"
16081620
""").format(
16091621
install_requirements_dir=install_requirements_dir,
1622+
requirements_file=requirements_file,
16101623
entry_point_command=" ".join(self.command),
16111624
entry_point=user_script,
16121625
)

‎sagemaker-core/src/sagemaker/core/spark/processing.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
from sagemaker.core import s3
4040
from sagemaker.core.local.image import _ecr_login_if_needed, _pull_image
4141
from sagemaker.core.processing import ProcessingInput, ProcessingOutput, ScriptProcessor
42+
from sagemaker.core.shapes import ProcessingS3Input, ProcessingS3Output
4243
from sagemaker.core.s3 import S3Uploader
4344
from sagemaker.core.helper.session_helper import Session
4445
from sagemaker.core.network import NetworkConfig
@@ -330,9 +331,12 @@ def _extend_processing_args(self, inputs, outputs, **kwargs):
330331
)
331332

332333
output = ProcessingOutput(
333-
source=_SparkProcessorBase._spark_event_log_default_local_path,
334-
destination=spark_event_logs_s3_uri,
335-
s3_upload_mode="Continuous",
334+
output_name="spark-event-logs",
335+
s3_output=ProcessingS3Output(
336+
s3_uri=spark_event_logs_s3_uri,
337+
local_path=_SparkProcessorBase._spark_event_log_default_local_path,
338+
s3_upload_mode="Continuous",
339+
),
336340
)
337341

338342
extended_outputs.append(output)
@@ -444,9 +448,13 @@ def _stage_configuration(self, configuration):
444448
)
445449

446450
conf_input = ProcessingInput(
447-
source=s3_uri,
448-
destination=f"{self._conf_container_base_path}{self._conf_container_input_name}",
449451
input_name=_SparkProcessorBase._conf_container_input_name,
452+
s3_input=ProcessingS3Input(
453+
s3_uri=s3_uri,
454+
local_path=f"{self._conf_container_base_path}{self._conf_container_input_name}",
455+
s3_data_type="S3Prefix",
456+
s3_input_mode="File",
457+
),
450458
)
451459
return conf_input
452460

@@ -473,6 +481,11 @@ def _stage_submit_deps(self, submit_deps, input_channel_name):
473481
)
474482
if not input_channel_name:
475483
raise ValueError("input_channel_name value may not be empty.")
484+
if not isinstance(submit_deps, (list, tuple)):
485+
raise ValueError(
486+
f"submit_deps must be a list of one or more paths, but got "
487+
f"{type(submit_deps).__name__}. {self._submit_deps_error_message}"
488+
)
476489

477490
use_input_channel = False
478491
spark_opt_s3_uris = []
@@ -544,15 +557,20 @@ def _stage_submit_deps(self, submit_deps, input_channel_name):
544557
# them to the Spark container and form the spark-submit option from a
545558
# combination of S3 URIs and container's local input path
546559
if use_input_channel:
560+
input_channel_local_path = f"{self._conf_container_base_path}{input_channel_name}"
547561
input_channel = ProcessingInput(
548-
source=input_channel_s3_uri,
549-
destination=f"{self._conf_container_base_path}{input_channel_name}",
550562
input_name=input_channel_name,
563+
s3_input=ProcessingS3Input(
564+
s3_uri=input_channel_s3_uri,
565+
local_path=input_channel_local_path,
566+
s3_data_type="S3Prefix",
567+
s3_input_mode="File",
568+
),
551569
)
552570
spark_opt = (
553-
Join(on=",", values=spark_opt_s3_uris + [input_channel.destination])
571+
Join(on=",", values=spark_opt_s3_uris + [input_channel_local_path])
554572
if spark_opt_s3_uris_has_pipeline_var
555-
else ",".join(spark_opt_s3_uris + [input_channel.destination])
573+
else ",".join(spark_opt_s3_uris + [input_channel_local_path])
556574
)
557575
# If no local files were uploaded, form the spark-submit option from a list of S3 URIs
558576
else:
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You
4+
# may not use this file except in compliance with the License. A copy of
5+
# the License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is
10+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11+
# ANY KIND, either express or implied. See the License for the specific
12+
# language governing permissions and limitations under the License.
13+
from __future__ import absolute_import
14+
15+
import pytest
16+
from unittest.mock import Mock, patch
17+
18+
from sagemaker.core.spark.processing import PySparkProcessor, _SparkProcessorBase
19+
from sagemaker.core.shapes import (
20+
ProcessingInput,
21+
ProcessingOutput,
22+
ProcessingS3Input,
23+
ProcessingS3Output,
24+
)
25+
26+
27+
@pytest.fixture
28+
def mock_session():
29+
session = Mock()
30+
session.boto_session = Mock()
31+
session.boto_session.region_name = "us-west-2"
32+
session.sagemaker_client = Mock()
33+
session.default_bucket = Mock(return_value="test-bucket")
34+
session.default_bucket_prefix = "sagemaker"
35+
session.expand_role = Mock(side_effect=lambda x: x)
36+
session.sagemaker_config = {}
37+
return session
38+
39+
40+
@pytest.fixture
41+
def pyspark_processor(mock_session):
42+
return PySparkProcessor(
43+
role="arn:aws:iam::123456789012:role/SageMakerRole",
44+
image_uri="test-spark-image:latest",
45+
instance_count=1,
46+
instance_type="ml.m5.xlarge",
47+
sagemaker_session=mock_session,
48+
)
49+
50+
51+
class TestStageSubmitDepsTypeValidation:
52+
"""#3809: a non-list submit dep must raise a clear ValueError, not iterate a string."""
53+
54+
def test_string_submit_deps_raises_clear_error(self, pyspark_processor):
55+
with pytest.raises(ValueError, match="submit_deps must be a list"):
56+
pyspark_processor._stage_submit_deps("s3://bucket/my.py", "py-files")
57+
58+
def test_list_submit_deps_does_not_raise_type_error(self, pyspark_processor):
59+
# A list of S3 URIs should be accepted and produce no input channel.
60+
input_channel, spark_opt = pyspark_processor._stage_submit_deps(
61+
["s3://bucket/my.py"], "py-files"
62+
)
63+
assert input_channel is None
64+
assert spark_opt == "s3://bucket/my.py"
65+
66+
67+
class TestSparkEventLogsOutputV3Shape:
68+
"""#6253: spark_event_logs_s3_uri output must use the V3 ProcessingOutput shape."""
69+
70+
def test_extend_processing_args_builds_v3_output(self, pyspark_processor):
71+
_, outputs = pyspark_processor._extend_processing_args(
72+
[], [], spark_event_logs_s3_uri="s3://bucket/spark-events/"
73+
)
74+
assert outputs is not None and len(outputs) == 1
75+
output = outputs[0]
76+
assert isinstance(output, ProcessingOutput)
77+
assert output.output_name == "spark-event-logs"
78+
assert isinstance(output.s3_output, ProcessingS3Output)
79+
assert output.s3_output.s3_uri == "s3://bucket/spark-events/"
80+
assert (
81+
output.s3_output.local_path == _SparkProcessorBase._spark_event_log_default_local_path
82+
)
83+
assert output.s3_output.s3_upload_mode == "Continuous"
84+
85+
86+
class TestStageSubmitDepsInputV3Shape:
87+
"""#6252: a local submit dep must build the V3 ProcessingInput shape."""
88+
89+
def test_local_dep_builds_v3_input(self, pyspark_processor):
90+
with (
91+
patch("sagemaker.core.spark.processing.os.path.isfile", return_value=True),
92+
patch("sagemaker.core.spark.processing.shutil.copy"),
93+
patch("sagemaker.core.spark.processing.os.listdir", return_value=["my.py"]),
94+
patch("sagemaker.core.spark.processing.S3Uploader.upload"),
95+
):
96+
input_channel, spark_opt = pyspark_processor._stage_submit_deps(
97+
["/local/path/my.py"], "py-files"
98+
)
99+
100+
assert isinstance(input_channel, ProcessingInput)
101+
assert input_channel.input_name == "py-files"
102+
assert isinstance(input_channel.s3_input, ProcessingS3Input)
103+
assert input_channel.s3_input.s3_data_type == "S3Prefix"
104+
assert input_channel.s3_input.s3_input_mode == "File"
105+
# The spark-submit option points at the container-local mount path.
106+
assert input_channel.s3_input.local_path in spark_opt

‎sagemaker-core/tests/unit/test_processing.py‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2292,3 +2292,99 @@ def test_sparkjar_processor_forwards_instance_preferences(self, mock_session):
22922292
sagemaker_session=mock_session,
22932293
)
22942294
assert processor.instance_preferences == self._PREFS
2295+
2296+
2297+
class TestFrameworkProcessorRequirements:
2298+
"""#5805: FrameworkProcessor.run(requirements=...) must be honored in the runproc script."""
2299+
2300+
def _make_processor(self, mock_session):
2301+
return FrameworkProcessor(
2302+
image_uri="test-image:latest",
2303+
role="arn:aws:iam::123456789012:role/SageMakerRole",
2304+
instance_count=1,
2305+
instance_type="ml.m5.xlarge",
2306+
command=["python3"],
2307+
sagemaker_session=mock_session,
2308+
)
2309+
2310+
def test_custom_requirements_file_is_used(self, mock_session):
2311+
processor = self._make_processor(mock_session)
2312+
script = processor._generate_framework_script(
2313+
"run_entrypoint.py", requirements="cpu-requirements.txt"
2314+
)
2315+
assert "if [[ -f 'cpu-requirements.txt' ]]; then" in script
2316+
assert "install_requirements.py cpu-requirements.txt" in script
2317+
assert "if [[ -f 'requirements.txt' ]]; then" not in script
2318+
2319+
def test_requirements_relative_path_uses_basename(self, mock_session):
2320+
processor = self._make_processor(mock_session)
2321+
script = processor._generate_framework_script(
2322+
"run_entrypoint.py", requirements="reqs/cpu-requirements.txt"
2323+
)
2324+
assert "if [[ -f 'cpu-requirements.txt' ]]; then" in script
2325+
2326+
def test_default_requirements_file_when_none(self, mock_session):
2327+
processor = self._make_processor(mock_session)
2328+
script = processor._generate_framework_script("run_entrypoint.py")
2329+
assert "if [[ -f 'requirements.txt' ]]; then" in script
2330+
assert "install_requirements.py requirements.txt" in script
2331+
2332+
2333+
class TestProcessorKmsKeyDefault:
2334+
"""#4874: Processor.run kms_key should default to the configured output_kms_key."""
2335+
2336+
def test_run_defaults_kms_key_to_output_kms_key(self, mock_session):
2337+
processor = Processor(
2338+
role="arn:aws:iam::123456789012:role/SageMakerRole",
2339+
image_uri="test-image:latest",
2340+
instance_count=1,
2341+
instance_type="ml.m5.xlarge",
2342+
output_kms_key="arn:aws:kms:us-west-2:123456789012:key/out-key",
2343+
sagemaker_session=mock_session,
2344+
)
2345+
2346+
captured = {}
2347+
2348+
def fake_normalize(**kwargs):
2349+
captured["kms_key"] = kwargs.get("kms_key")
2350+
return [], []
2351+
2352+
with (
2353+
patch.object(processor, "_normalize_args", side_effect=fake_normalize),
2354+
patch.object(processor, "_start_new", return_value=Mock()),
2355+
patch(
2356+
"sagemaker.core.processing.check_and_get_run_experiment_config",
2357+
side_effect=lambda x: x,
2358+
),
2359+
):
2360+
processor.run(wait=False, logs=False)
2361+
2362+
assert captured["kms_key"] == "arn:aws:kms:us-west-2:123456789012:key/out-key"
2363+
2364+
def test_run_explicit_kms_key_wins(self, mock_session):
2365+
processor = Processor(
2366+
role="arn:aws:iam::123456789012:role/SageMakerRole",
2367+
image_uri="test-image:latest",
2368+
instance_count=1,
2369+
instance_type="ml.m5.xlarge",
2370+
output_kms_key="arn:aws:kms:us-west-2:123456789012:key/out-key",
2371+
sagemaker_session=mock_session,
2372+
)
2373+
2374+
captured = {}
2375+
2376+
def fake_normalize(**kwargs):
2377+
captured["kms_key"] = kwargs.get("kms_key")
2378+
return [], []
2379+
2380+
with (
2381+
patch.object(processor, "_normalize_args", side_effect=fake_normalize),
2382+
patch.object(processor, "_start_new", return_value=Mock()),
2383+
patch(
2384+
"sagemaker.core.processing.check_and_get_run_experiment_config",
2385+
side_effect=lambda x: x,
2386+
),
2387+
):
2388+
processor.run(wait=False, logs=False, kms_key="arn:aws:kms:us-west-2:1:key/explicit")
2389+
2390+
assert captured["kms_key"] == "arn:aws:kms:us-west-2:1:key/explicit"

0 commit comments

Comments
 (0)