Skip to content

Commit 7e84ec9

Browse files
mohamedzeidan2021Mohamed Zeidan
andauthored
fix: do not apply user ignore_patterns to the sm_drivers driver channel (#6316)
* fix: do not apply user ignore_patterns to the sm_drivers driver channel ModelTrainer forwarded the user's SourceCode.ignore_patterns to the SDK-owned sm_drivers channel. Patterns meant for the user's source_dir (e.g. "scripts", "environment") then stripped the driver's own scripts/environment.py during packaging, so the container bootstrap failed with "sm_drivers/scripts/environment.py: No such file or directory". Stop passing ignore_patterns on the driver channel; they still apply to the source_dir channel. Fixes #5493 * docs: correct create_input_data_channel ignore_patterns docstring (no default filter) --------- Co-authored-by: Mohamed Zeidan <zeidmo@amazon.com>
1 parent e1e73de commit 7e84ec9

2 files changed

Lines changed: 49 additions & 4 deletions

File tree

‎sagemaker-train/src/sagemaker/train/model_trainer.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -734,12 +734,15 @@ def _create_training_job_args(
734734
self._write_source_code_json(tmp_dir=self._temp_code_dir, source_code=self.source_code)
735735
self._write_distributed_json(tmp_dir=self._temp_code_dir, distributed=self.distributed)
736736

737-
# Create an input channel for drivers packaged by the sdk
737+
# Create an input channel for drivers packaged by the sdk.
738+
# Do NOT apply the user's source_code.ignore_patterns here: this directory is
739+
# SDK-owned driver content (e.g. scripts/environment.py), and user patterns such as
740+
# "scripts" or "environment" would strip files the container bootstrap requires,
741+
# causing "sm_drivers/scripts/environment.py: No such file or directory" (issue #5493).
738742
sm_drivers_channel = self.create_input_data_channel(
739743
channel_name=SM_DRIVERS,
740744
data_source=self._temp_code_dir.name,
741745
key_prefix=input_data_key_prefix,
742-
ignore_patterns=self.source_code.ignore_patterns,
743746
instance_group_names=managed_channel_instance_group_names,
744747
)
745748
final_input_data_config.append(sm_drivers_channel)
@@ -1051,8 +1054,7 @@ def create_input_data_channel(
10511054
``s3://<default_bucket_path>/<key_prefix>/<channel_name>/``
10521055
ignore_patterns: (Optional[List[str]]) :
10531056
The ignore patterns to ignore specific files/folders when uploading to S3.
1054-
If not specified, default to:
1055-
['.env', '.git', '__pycache__', '.DS_Store', '.cache', '.ipynb_checkpoints'].
1057+
If not specified, no files are filtered and the data source is uploaded as-is.
10561058
instance_group_names: (Optional[List[str]]) :
10571059
The names of the instance groups (for heterogeneous clusters) that this
10581060
channel's data should be assigned to. Only applied when the channel is

‎sagemaker-train/tests/unit/train/test_model_trainer.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@
5252
SOURCE_CODE_JSON,
5353
TRAIN_SCRIPT,
5454
SM_RECIPE_CONTAINER_PATH,
55+
SM_CODE,
56+
SM_DRIVERS,
5557
)
5658
from sagemaker.train.configs import (
5759
Compute,
@@ -254,6 +256,47 @@ def test_model_trainer_param_validation(test_case, modules_session):
254256
assert trainer.base_job_name == DEFAULT_BASE_NAME
255257

256258

259+
@patch("sagemaker.train.model_trainer.TrainingJob")
260+
@patch("sagemaker.train.model_trainer.ModelTrainer.create_input_data_channel")
261+
def test_sm_drivers_channel_ignores_user_ignore_patterns(
262+
mock_create_input_data_channel, mock_training_job, modules_session
263+
):
264+
"""Regression for #5493.
265+
266+
The user's ``ignore_patterns`` apply only to their own source_dir channel. They must NOT be
267+
forwarded to the SDK-owned ``sm_drivers`` driver channel -- patterns like ``"scripts"`` or
268+
``"environment"`` would strip the driver's own ``scripts/environment.py`` and break the
269+
container bootstrap with "sm_drivers/scripts/environment.py: No such file or directory".
270+
"""
271+
user_patterns = ["scripts", "environment", "data"]
272+
trainer = ModelTrainer(
273+
training_image=DEFAULT_IMAGE,
274+
role=DEFAULT_ROLE,
275+
compute=DEFAULT_COMPUTE_CONFIG,
276+
stopping_condition=DEFAULT_STOPPING_CONDITION,
277+
output_data_config=DEFAULT_OUTPUT_DATA_CONFIG,
278+
source_code=SourceCode(
279+
source_dir=DEFAULT_SOURCE_DIR,
280+
entry_script="custom_script.py",
281+
ignore_patterns=user_patterns,
282+
),
283+
sagemaker_session=modules_session,
284+
)
285+
286+
trainer.train()
287+
288+
calls_by_channel = {
289+
call.kwargs.get("channel_name"): call.kwargs
290+
for call in mock_create_input_data_channel.call_args_list
291+
}
292+
assert SM_CODE in calls_by_channel, "source_dir channel was not created"
293+
assert SM_DRIVERS in calls_by_channel, "sm_drivers channel was not created"
294+
# User patterns still apply to the user's own source_dir channel.
295+
assert calls_by_channel[SM_CODE].get("ignore_patterns") == user_patterns
296+
# ...but must NOT be applied to the SDK-owned driver channel.
297+
assert calls_by_channel[SM_DRIVERS].get("ignore_patterns") is None
298+
299+
257300
@patch("sagemaker.train.model_trainer.TrainingJob")
258301
def test_train_with_default_params(mock_training_job, model_trainer):
259302
model_trainer.train()

0 commit comments

Comments
 (0)