From 622d110e6c596daa62807fd80d25809fc29cb59e Mon Sep 17 00:00:00 2001 From: Mohamed Zeidan Date: Thu, 24 Sep 2026 14:15:01 -0700 Subject: [PATCH 1/2] fix: do not apply user ignore_patterns to the sm_drivers driver channel ModelTrainer forwarded the user's SourceCode.ignore_patterns to the SDK-owned sm_drivers channel. Patterns meant for the user's source_dir (e.g. "scripts", "environment") then stripped the driver's own scripts/environment.py during packaging, so the container bootstrap failed with "sm_drivers/scripts/environment.py: No such file or directory". Stop passing ignore_patterns on the driver channel; they still apply to the source_dir channel. Fixes #5493 --- .../src/sagemaker/train/model_trainer.py | 7 ++- .../tests/unit/train/test_model_trainer.py | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/model_trainer.py b/sagemaker-train/src/sagemaker/train/model_trainer.py index ce954036f6..9e0a3c4da7 100644 --- a/sagemaker-train/src/sagemaker/train/model_trainer.py +++ b/sagemaker-train/src/sagemaker/train/model_trainer.py @@ -734,12 +734,15 @@ def _create_training_job_args( self._write_source_code_json(tmp_dir=self._temp_code_dir, source_code=self.source_code) self._write_distributed_json(tmp_dir=self._temp_code_dir, distributed=self.distributed) - # Create an input channel for drivers packaged by the sdk + # Create an input channel for drivers packaged by the sdk. + # Do NOT apply the user's source_code.ignore_patterns here: this directory is + # SDK-owned driver content (e.g. scripts/environment.py), and user patterns such as + # "scripts" or "environment" would strip files the container bootstrap requires, + # causing "sm_drivers/scripts/environment.py: No such file or directory" (issue #5493). sm_drivers_channel = self.create_input_data_channel( channel_name=SM_DRIVERS, data_source=self._temp_code_dir.name, key_prefix=input_data_key_prefix, - ignore_patterns=self.source_code.ignore_patterns, instance_group_names=managed_channel_instance_group_names, ) final_input_data_config.append(sm_drivers_channel) diff --git a/sagemaker-train/tests/unit/train/test_model_trainer.py b/sagemaker-train/tests/unit/train/test_model_trainer.py index 6022c60098..4c63619b45 100644 --- a/sagemaker-train/tests/unit/train/test_model_trainer.py +++ b/sagemaker-train/tests/unit/train/test_model_trainer.py @@ -52,6 +52,8 @@ SOURCE_CODE_JSON, TRAIN_SCRIPT, SM_RECIPE_CONTAINER_PATH, + SM_CODE, + SM_DRIVERS, ) from sagemaker.train.configs import ( Compute, @@ -254,6 +256,47 @@ def test_model_trainer_param_validation(test_case, modules_session): assert trainer.base_job_name == DEFAULT_BASE_NAME +@patch("sagemaker.train.model_trainer.TrainingJob") +@patch("sagemaker.train.model_trainer.ModelTrainer.create_input_data_channel") +def test_sm_drivers_channel_ignores_user_ignore_patterns( + mock_create_input_data_channel, mock_training_job, modules_session +): + """Regression for #5493. + + The user's ``ignore_patterns`` apply only to their own source_dir channel. They must NOT be + forwarded to the SDK-owned ``sm_drivers`` driver channel -- patterns like ``"scripts"`` or + ``"environment"`` would strip the driver's own ``scripts/environment.py`` and break the + container bootstrap with "sm_drivers/scripts/environment.py: No such file or directory". + """ + user_patterns = ["scripts", "environment", "data"] + trainer = ModelTrainer( + training_image=DEFAULT_IMAGE, + role=DEFAULT_ROLE, + compute=DEFAULT_COMPUTE_CONFIG, + stopping_condition=DEFAULT_STOPPING_CONDITION, + output_data_config=DEFAULT_OUTPUT_DATA_CONFIG, + source_code=SourceCode( + source_dir=DEFAULT_SOURCE_DIR, + entry_script="custom_script.py", + ignore_patterns=user_patterns, + ), + sagemaker_session=modules_session, + ) + + trainer.train() + + calls_by_channel = { + call.kwargs.get("channel_name"): call.kwargs + for call in mock_create_input_data_channel.call_args_list + } + assert SM_CODE in calls_by_channel, "source_dir channel was not created" + assert SM_DRIVERS in calls_by_channel, "sm_drivers channel was not created" + # User patterns still apply to the user's own source_dir channel. + assert calls_by_channel[SM_CODE].get("ignore_patterns") == user_patterns + # ...but must NOT be applied to the SDK-owned driver channel. + assert calls_by_channel[SM_DRIVERS].get("ignore_patterns") is None + + @patch("sagemaker.train.model_trainer.TrainingJob") def test_train_with_default_params(mock_training_job, model_trainer): model_trainer.train() From b85dcd139f307219584bdddfc8e2f6e44f8cfcb7 Mon Sep 17 00:00:00 2001 From: Mohamed Zeidan Date: Thu, 24 Sep 2026 14:17:45 -0700 Subject: [PATCH 2/2] docs: correct create_input_data_channel ignore_patterns docstring (no default filter) --- sagemaker-train/src/sagemaker/train/model_trainer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/model_trainer.py b/sagemaker-train/src/sagemaker/train/model_trainer.py index 9e0a3c4da7..9ce9958c8a 100644 --- a/sagemaker-train/src/sagemaker/train/model_trainer.py +++ b/sagemaker-train/src/sagemaker/train/model_trainer.py @@ -1054,8 +1054,7 @@ def create_input_data_channel( ``s3://///`` ignore_patterns: (Optional[List[str]]) : The ignore patterns to ignore specific files/folders when uploading to S3. - If not specified, default to: - ['.env', '.git', '__pycache__', '.DS_Store', '.cache', '.ipynb_checkpoints']. + If not specified, no files are filtered and the data source is uploaded as-is. instance_group_names: (Optional[List[str]]) : The names of the instance groups (for heterogeneous clusters) that this channel's data should be assigned to. Only applied when the channel is