Skip to content

Commit cd42fee

Browse files
author
Sagar Dubey
committed
fix(tgi): honor S3 model_path as weight source for TGI builds
ModelBuilder with ModelServer.TGI silently ignored an S3 weight source (model_path="s3://..." or s3_model_data_url="s3://..."): it created a literal local "s3:/..." directory and set HF_MODEL_ID to the HF repo id, so the container always downloaded weights from huggingface.co. _build_for_tgi now detects an S3 weight source, skips the local mkdir for it, attaches the S3 prefix as an uncompressed ModelDataSource, and sets HF_MODEL_ID=/opt/ml/model and HF_HUB_OFFLINE=1 (via setdefault, preserving any user-supplied HF_MODEL_ID). Genuine local paths, HF-Hub downloads, JumpStart, and all non-TGI servers are unchanged. Also fixes two defects surfaced during real deployment: - HF_HUB_OFFLINE was reset to "0" at the end of the build; it now stays "1" for the S3-mounted path so TGI loads from /opt/ml/model. - The uncompressed ModelDataSource S3Uri could end in "//" when the input prefix already had a trailing slash; normalized to exactly one slash so S3Prefix matching finds the weight objects. Adds unit and regression tests for all of the above.
1 parent db63167 commit cd42fee

4 files changed

Lines changed: 641 additions & 4 deletions

File tree

‎sagemaker-serve/src/sagemaker/serve/model_builder_servers.py‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
# SageMaker core imports
2626
from sagemaker.core.resources import Model, Endpoint
2727
from sagemaker.core.utils.utils import logger
28+
from sagemaker.core.common_utils import _is_s3_uri
2829

2930

3031
# SageMaker serve imports
@@ -205,11 +206,30 @@ def _build_for_tgi(self) -> Model:
205206

206207
from sagemaker.serve.model_server.tgi.prepare import _create_dir_structure
207208

208-
_create_dir_structure(self.model_path)
209+
# Detect an S3 weight source (model_path="s3://..." or
210+
# s3_model_data_url="s3://...") before any local directory is created. TGI's
211+
# HF_MODEL_ID does not accept an S3 URI, so an S3 source is attached as an
212+
# uncompressed ModelDataSource (mounted at /opt/ml/model) instead of being
213+
# downloaded from the HuggingFace Hub.
214+
s3_model_source = None
215+
if _is_s3_uri(self.model_path):
216+
s3_model_source = self.model_path
217+
elif _is_s3_uri(self.s3_model_data_url):
218+
s3_model_source = self.s3_model_data_url
219+
220+
# Skip the local mkdir for an S3 source so we do not create a literal
221+
# local "s3:/..." directory tree; only create it for genuine local paths.
222+
if not s3_model_source:
223+
_create_dir_structure(self.model_path)
209224

210225
if isinstance(self.model, str) and not self._is_jumpstart_model_id():
211226
# Configure HuggingFace model for TGI
212-
self.env_vars.setdefault("HF_MODEL_ID", self.model)
227+
if s3_model_source:
228+
# Weights are mounted at /opt/ml/model; do not download from the Hub.
229+
self.env_vars.setdefault("HF_MODEL_ID", "/opt/ml/model")
230+
self.env_vars.setdefault("HF_HUB_OFFLINE", "1")
231+
else:
232+
self.env_vars.setdefault("HF_MODEL_ID", self.model)
213233

214234
self.hf_model_config = _get_model_config_properties_from_hf(
215235
self.model, self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
@@ -252,6 +272,11 @@ def _build_for_tgi(self) -> Model:
252272
if not self._optimizing:
253273
if self.mode in LOCAL_MODES:
254274
self._prepare_for_mode(should_upload_artifacts=True)
275+
elif s3_model_source:
276+
# Route the S3 weight source through _prepare_for_mode so the
277+
# _upload_tgi_artifacts S3 branch builds the uncompressed
278+
# ModelDataSource (CompressionType="None", S3DataType="S3Prefix").
279+
self.s3_model_data_url, _ = self._prepare_for_mode(model_path=s3_model_source)
255280
else:
256281
self.s3_model_data_url, _ = self._prepare_for_mode()
257282

@@ -284,7 +309,10 @@ def _build_for_tgi(self) -> Model:
284309

285310
model = self._create_model()
286311

287-
if "HF_HUB_OFFLINE" in self.env_vars:
312+
# Reset the in-memory HF_HUB_OFFLINE flag after the container is built,
313+
# EXCEPT when weights are mounted from S3: those must stay offline so TGI
314+
# loads from /opt/ml/model instead of phoning home to the HuggingFace Hub.
315+
if "HF_HUB_OFFLINE" in self.env_vars and not s3_model_source:
288316
self.env_vars.update({"HF_HUB_OFFLINE": "0"})
289317

290318
return model

‎sagemaker-serve/src/sagemaker/serve/model_server/tgi/server.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,11 @@ def _upload_tgi_artifacts(
131131
"S3DataSource": {
132132
"CompressionType": "None",
133133
"S3DataType": "S3Prefix",
134-
"S3Uri": model_data_url + "/",
134+
# Normalize trailing slashes so an S3Prefix URI has exactly one.
135+
# A user-supplied weight prefix may already end in "/", and a
136+
# doubled "s3://.../prefix//" would not match the actual object
137+
# keys under "s3://.../prefix/" with S3Prefix matching.
138+
"S3Uri": model_data_url.rstrip("/") + "/",
135139
}
136140
}
137141
if model_data_url

‎sagemaker-serve/tests/unit/model_server/test_tgi_server.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,33 @@ def test_upload_tgi_artifacts_with_s3_path(self, mock_is_s3):
127127
self.assertIsNotNone(model_data)
128128
self.assertEqual(model_data["S3DataSource"]["S3Uri"], "s3://bucket/model/")
129129

130+
@patch("sagemaker.serve.model_server.tgi.server._is_s3_uri")
131+
def test_upload_tgi_artifacts_s3_path_trailing_slash_normalized(self, mock_is_s3):
132+
"""An S3 model_path that already ends in "/" must not yield a doubled "//".
133+
134+
With S3DataType="S3Prefix", SageMaker does literal prefix matching, so a
135+
doubled "s3://bucket/model//" would fail to match the actual object keys
136+
under "s3://bucket/model/" and nothing would mount into /opt/ml/model.
137+
Regression test for the trailing-slash deploy-breaker.
138+
"""
139+
from sagemaker.serve.model_server.tgi.server import SageMakerTgiServing
140+
141+
server = SageMakerTgiServing()
142+
mock_is_s3.return_value = True
143+
mock_session = Mock()
144+
145+
model_data, _ = server._upload_tgi_artifacts(
146+
model_path="s3://bucket/model/",
147+
sagemaker_session=mock_session,
148+
jumpstart=False,
149+
should_upload_artifacts=False,
150+
)
151+
152+
self.assertIsNotNone(model_data)
153+
# Exactly one trailing slash, regardless of the input's trailing slash.
154+
self.assertEqual(model_data["S3DataSource"]["S3Uri"], "s3://bucket/model/")
155+
self.assertNotIn("//", model_data["S3DataSource"]["S3Uri"].split("://", 1)[1])
156+
130157
@patch("sagemaker.serve.model_server.tgi.server._is_s3_uri")
131158
def test_upload_tgi_artifacts_jumpstart(self, mock_is_s3):
132159
"""Test _upload_tgi_artifacts with jumpstart=True."""

0 commit comments

Comments
 (0)