Skip to content

Commit c29d1cf

Browse files
author
Sagar Dubey
committed
fix(tgi): distinguish S3 model source from upload destination
1 parent 394adb8 commit c29d1cf

2 files changed

Lines changed: 35 additions & 28 deletions

File tree

‎sagemaker-serve/src/sagemaker/serve/model_builder_servers.py‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -221,16 +221,11 @@ def _build_for_tgi(self) -> Model:
221221

222222
from sagemaker.serve.model_server.tgi.prepare import _create_dir_structure
223223

224-
# Detect an S3 weight source (model_path="s3://..." or
225-
# s3_model_data_url="s3://...") before any local directory is created. TGI's
226-
# HF_MODEL_ID does not accept an S3 URI, so an S3 source is attached as an
227-
# uncompressed ModelDataSource (mounted at /opt/ml/model) instead of being
228-
# downloaded from the HuggingFace Hub.
229-
s3_model_source = None
230-
if _is_s3_uri(self.model_path):
231-
s3_model_source = self.model_path
232-
elif _is_s3_uri(self.s3_model_data_url):
233-
s3_model_source = self.s3_model_data_url
224+
# Detect an S3 weight source from model_path before any local directory is
225+
# created. TGI's HF_MODEL_ID does not accept an S3 URI, so an S3 source is
226+
# attached as an uncompressed ModelDataSource (mounted at /opt/ml/model)
227+
# instead of being downloaded from the HuggingFace Hub.
228+
s3_model_source = self.model_path if _is_s3_uri(self.model_path) else None
234229

235230
# Skip the local mkdir for an S3 source so we do not create a literal
236231
# local "s3:/..." directory tree; only create it for genuine local paths.

‎sagemaker-serve/tests/unit/servers/test_model_builder_servers.py‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -609,7 +609,7 @@ def test_fixed_s3_hf_hub_offline_survives_post_build_reset(
609609
@patch.object(MockModelBuilderServers, "_auto_detect_image_uri")
610610
@patch.object(MockModelBuilderServers, "_prepare_for_mode")
611611
@patch.object(MockModelBuilderServers, "_create_model")
612-
def test_fixed_s3_model_data_url_routed_to_s3_branch(
612+
def test_generated_s3_model_data_url_remains_upload_destination(
613613
self,
614614
mock_create,
615615
mock_prepare,
@@ -623,29 +623,30 @@ def test_fixed_s3_model_data_url_routed_to_s3_branch(
623623
mock_tp,
624624
mock_gpu,
625625
):
626-
"""Property 2 (clause 2.4): an s3_model_data_url weight source is honored.
626+
"""An auto-generated S3 destination does not select TGI model weights.
627627
628-
On the FIXED code the non-local branch detects the S3 weight source
629-
supplied via ``s3_model_data_url`` and calls
630-
``_prepare_for_mode(model_path=<s3 url>)`` so it reaches the
631-
``_upload_tgi_artifacts`` S3 upload branch, and the container env
632-
(captured at ``_create_model`` time) has ``HF_MODEL_ID=/opt/ml/model``.
633-
EXPECTED OUTCOME: this test PASSES on fixed code.
628+
``_get_serve_setting()`` populates ``s3_model_data_url`` for a normal
629+
HuggingFace Hub build even when the user did not provide an S3 weight
630+
source. The destination must not enable offline mode, replace the repo
631+
id, or be routed to ``_prepare_for_mode`` as ``model_path``.
634632
635-
Validates: Requirements 2.4
633+
Validates: Requirements 2.5, 3.2, 3.3, 4.2
636634
"""
637635
mock_js.return_value = False
638636
mock_nb.return_value = None
639637
mock_hf_config.return_value = {"model_type": "gpt2"}
640638
mock_tgi_config.return_value = ({"MAX_INPUT_LENGTH": "1024"}, 512)
641639
mock_gpu.return_value = 1
642640
mock_tp.return_value = 1
643-
mock_prepare.return_value = ("s3://bucket/model.tar.gz", None)
641+
mock_create.return_value = Mock()
644642
self.builder.mode = Mode.SAGEMAKER_ENDPOINT
645643
self.builder.model = "org/model"
646-
# model_path stays the genuine local default; the S3 weight source is
647-
# supplied via s3_model_data_url.
648-
self.builder.s3_model_data_url = "s3://bucket/weights/"
644+
self.builder.model_path = "/tmp/local-model"
645+
generated_destination = (
646+
"s3://default-bucket/model-builder/model/0123456789abcdef0123456789abcdef/"
647+
)
648+
self.builder.s3_model_data_url = generated_destination
649+
mock_prepare.return_value = (generated_destination, None)
649650

650651
captured_env = {}
651652

@@ -657,11 +658,22 @@ def _capture_container_env():
657658

658659
self.builder._build_for_tgi()
659660

660-
# Property 2 (2.4): the s3_model_data_url is routed through the S3
661-
# upload branch via _prepare_for_mode(model_path=...).
662-
mock_prepare.assert_called_once_with(model_path="s3://bucket/weights/")
663-
self.assertEqual(captured_env["HF_MODEL_ID"], "/opt/ml/model")
664-
self.assertEqual(captured_env["HF_HUB_OFFLINE"], "1")
661+
mock_prepare.assert_called_once()
662+
routed_model_path = mock_prepare.call_args.kwargs.get("model_path")
663+
self.assertEqual(
664+
{
665+
"hf_model_id": captured_env.get("HF_MODEL_ID"),
666+
"hf_hub_offline": captured_env.get("HF_HUB_OFFLINE"),
667+
"routed_model_path": routed_model_path,
668+
"s3_source_created": routed_model_path is not None,
669+
},
670+
{
671+
"hf_model_id": "org/model",
672+
"hf_hub_offline": None,
673+
"routed_model_path": None,
674+
"s3_source_created": False,
675+
},
676+
)
665677

666678
@patch("sagemaker.serve.model_builder_servers._get_gpu_info")
667679
@patch("sagemaker.serve.model_builder_servers._get_default_tensor_parallel_degree")

0 commit comments

Comments
 (0)