Skip to content

Commit 80775f7

Browse files
committed
feat(serve): route HuggingFace models to vLLM/SGLang/vLLM-omni in ModelBuilder
Add ModelServer.VLLM/SGLANG/VLLM_OMNI and teach ModelBuilder's auto-detection to select the new HuggingFace DLCs: text-generation now defaults to vLLM (replacing archived TGI), multimodal tasks route to vLLM-omni, and SGLang is reachable via explicit model_server. TEI/transformers routing is unchanged. Signed-off-by: DWarez <dario.salvati@huggingface.co>
1 parent 96319e1 commit 80775f7

8 files changed

Lines changed: 299 additions & 25 deletions

File tree

‎sagemaker-serve/src/sagemaker/serve/constants.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,17 @@
6565
ModelServer.TGI,
6666
ModelServer.TEI,
6767
ModelServer.SMD,
68+
ModelServer.VLLM,
69+
ModelServer.SGLANG,
70+
ModelServer.VLLM_OMNI,
71+
}
72+
73+
# HuggingFace pipeline tags that should route to the vLLM-omni (multimodal) server.
74+
OMNI_TASKS: Set[str] = {
75+
"image-text-to-text",
76+
"any-to-any",
77+
"visual-question-answering",
78+
"audio-text-to-text",
6879
}
6980

7081
# ========================================

‎sagemaker-serve/src/sagemaker/serve/model_builder.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,12 @@
135135
ENDPOINT_CONFIG_ASYNC_KMS_KEY_ID_PATH,
136136
MODEL_CONTAINERS_PATH,
137137
)
138-
from sagemaker.serve.constants import LOCAL_MODES, SUPPORTED_MODEL_SERVERS, Framework
138+
from sagemaker.serve.constants import (
139+
LOCAL_MODES,
140+
SUPPORTED_MODEL_SERVERS,
141+
OMNI_TASKS,
142+
Framework,
143+
)
139144
from sagemaker.core.workflow.pipeline_context import PipelineSession, runnable_by_pipeline
140145
from sagemaker.core import fw_utils
141146
from sagemaker.core.helper.session_helper import container_def
@@ -2692,8 +2697,14 @@ def _build_single_modelbuilder(
26922697
if self.schema_builder is None and model_task is not None:
26932698
self._hf_schema_builder_init(model_task)
26942699

2700+
# Task-based auto-selection. SGLang is not auto-selected by task; it is
2701+
# opt-in only via model_server=ModelServer.SGLANG, which is handled earlier
2702+
# by the _build_for_model_server() short-circuit above.
26952703
if model_task == "text-generation":
2696-
self.built_model = self._build_for_tgi()
2704+
self.built_model = self._build_for_vllm()
2705+
return self.built_model
2706+
elif model_task in OMNI_TASKS:
2707+
self.built_model = self._build_for_vllm_omni()
26972708
return self.built_model
26982709
elif model_task in ["sentence-similarity", "feature-extraction"]:
26992710
self.built_model = self._build_for_tei()

‎sagemaker-serve/src/sagemaker/serve/model_builder_servers.py‎

Lines changed: 108 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
and deploying machine learning models across different model servers and deployment modes.
1717
It supports various frameworks including PyTorch, TensorFlow, HuggingFace, XGBoost, and more.
1818
"""
19+
1920
from __future__ import absolute_import, annotations
2021

2122
import os
@@ -36,7 +37,10 @@
3637
_get_gpu_info_fallback,
3738
)
3839
from sagemaker.serve.utils.hf_utils import _get_model_config_properties_from_hf
39-
from sagemaker.serve.detector.image_detector import _get_model_base, _detect_framework_and_version
40+
from sagemaker.serve.detector.image_detector import (
41+
_get_model_base,
42+
_detect_framework_and_version,
43+
)
4044
from sagemaker.serve.detector.pickler import save_pkl
4145
from sagemaker.serve.utils.types import ModelServer
4246

@@ -48,11 +52,15 @@
4852
from sagemaker.serve.model_server.tgi.utils import _get_default_tgi_configurations
4953
from sagemaker.serve.model_server.tgi.prepare import prepare_tgi_js_resources
5054
from sagemaker.serve.model_server.djl_serving.prepare import prepare_djl_js_resources
51-
from sagemaker.serve.model_server.multi_model_server.prepare import prepare_mms_js_resources
55+
from sagemaker.serve.model_server.multi_model_server.prepare import (
56+
prepare_mms_js_resources,
57+
)
5258

5359
# Model server preparation
5460
from sagemaker.serve.model_server.torchserve.prepare import prepare_for_torchserve
55-
from sagemaker.serve.model_server.tensorflow_serving.prepare import prepare_for_tf_serving
61+
from sagemaker.serve.model_server.tensorflow_serving.prepare import (
62+
prepare_for_tf_serving,
63+
)
5664
from sagemaker.serve.model_server.smd.prepare import prepare_for_smd
5765
from sagemaker.serve.model_server.multi_model_server.prepare import prepare_for_mms
5866

@@ -70,7 +78,6 @@
7078

7179

7280
class _ModelBuilderServers(object):
73-
7481
def _build_for_model_server(self) -> Model:
7582
"""Build model using explicit model server configuration.
7683
@@ -109,6 +116,12 @@ def _build_for_model_server(self) -> Model:
109116
return self._build_for_tei()
110117
elif self.model_server == ModelServer.TGI:
111118
return self._build_for_tgi()
119+
elif self.model_server == ModelServer.VLLM:
120+
return self._build_for_vllm()
121+
elif self.model_server == ModelServer.SGLANG:
122+
return self._build_for_sglang()
123+
elif self.model_server == ModelServer.VLLM_OMNI:
124+
return self._build_for_vllm_omni()
112125
elif self.model_server == ModelServer.MMS:
113126
return self._build_for_transformers()
114127
elif self.model_server == ModelServer.SMD:
@@ -289,6 +302,97 @@ def _build_for_tgi(self) -> Model:
289302

290303
return model
291304

305+
def _build_for_hf_server(self, model_server: ModelServer) -> Model:
306+
"""Build a HuggingFace model for a hub-download serving container.
307+
308+
Generic build path shared by the vLLM, SGLang, and vLLM-omni servers. It
309+
configures the container to pull the model directly from the HuggingFace Hub
310+
(HF_MODEL_ID), resolves the appropriate DLC image via _auto_detect_image_uri,
311+
and prepares the model for the selected mode. Server-specific tuning (sharding,
312+
tensor parallelism, MAX_* limits, etc.) is intentionally left to the container
313+
defaults and will be added in a follow-up.
314+
315+
Args:
316+
model_server: The HuggingFace serving model server to build for
317+
(VLLM, SGLANG, or VLLM_OMNI).
318+
319+
Returns:
320+
Model: Configured model ready for deployment.
321+
"""
322+
self.secret_key = ""
323+
self.model_server = model_server
324+
325+
# Use notebook instance type if available
326+
nb_instance = _get_nb_instance()
327+
if nb_instance and not self._user_provided_instance_type:
328+
self.instance_type = nb_instance
329+
logger.debug(f"Using detected notebook instance type: {nb_instance}")
330+
331+
from sagemaker.serve.model_server.tgi.prepare import _create_dir_structure
332+
333+
_create_dir_structure(self.model_path)
334+
335+
if isinstance(self.model, str) and not self._is_jumpstart_model_id():
336+
# These containers download the model directly from the HuggingFace Hub
337+
# Todo: missing something?
338+
self.env_vars.setdefault("HF_MODEL_ID", self.model)
339+
340+
if self.env_vars.get("HUGGING_FACE_HUB_TOKEN"):
341+
self.env_vars["HF_TOKEN"] = self.env_vars.get("HUGGING_FACE_HUB_TOKEN")
342+
343+
self.s3_upload_path = None
344+
345+
self._auto_detect_image_uri()
346+
347+
if not self._optimizing:
348+
if self.mode in LOCAL_MODES:
349+
self._prepare_for_mode(should_upload_artifacts=True)
350+
else:
351+
self.s3_model_data_url, _ = self._prepare_for_mode()
352+
353+
# Cache management based on mode
354+
if self.mode in LOCAL_MODES:
355+
self.env_vars.update({"HF_HUB_OFFLINE": "1"})
356+
else:
357+
self.env_vars["HF_HOME"] = "/tmp"
358+
self.env_vars["HUGGINGFACE_HUB_CACHE"] = "/tmp"
359+
360+
if self.mode == Mode.SAGEMAKER_ENDPOINT and not self.instance_type:
361+
raise ValueError(
362+
"Instance type must be provided when building for SageMaker Endpoint mode."
363+
)
364+
365+
model = self._create_model()
366+
367+
if "HF_HUB_OFFLINE" in self.env_vars:
368+
self.env_vars.update({"HF_HUB_OFFLINE": "0"})
369+
370+
return model
371+
372+
def _build_for_vllm(self) -> Model:
373+
"""Build a HuggingFace model for the vLLM serving container.
374+
375+
Returns:
376+
Model: Configured model ready for vLLM deployment.
377+
"""
378+
return self._build_for_hf_server(ModelServer.VLLM)
379+
380+
def _build_for_sglang(self) -> Model:
381+
"""Build a HuggingFace model for the SGLang serving container.
382+
383+
Returns:
384+
Model: Configured model ready for SGLang deployment.
385+
"""
386+
return self._build_for_hf_server(ModelServer.SGLANG)
387+
388+
def _build_for_vllm_omni(self) -> Model:
389+
"""Build a HuggingFace model for the vLLM-omni (multimodal) serving container.
390+
391+
Returns:
392+
Model: Configured model ready for vLLM-omni deployment.
393+
"""
394+
return self._build_for_hf_server(ModelServer.VLLM_OMNI)
395+
292396
def _build_for_djl(self) -> Model:
293397
"""Build model for DJL Serving deployment.
294398

‎sagemaker-serve/src/sagemaker/serve/model_builder_utils.py‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ def build(self):
6262
from sagemaker.serve.compute_resource_requirements import ResourceRequirements
6363
from sagemaker.serve.constants import (
6464
DEFAULT_SERIALIZERS_BY_FRAMEWORK,
65+
OMNI_TASKS,
6566
Framework,
6667
)
6768
from sagemaker.serve.builder.schema_builder import SchemaBuilder
@@ -675,7 +676,9 @@ def _detect_huggingface_image(self) -> None:
675676
model_task = hf_model_md.get("pipeline_tag")
676677

677678
if model_task == "text-generation":
678-
effective_model_server = ModelServer.TGI
679+
effective_model_server = ModelServer.VLLM
680+
elif model_task in OMNI_TASKS:
681+
effective_model_server = ModelServer.VLLM_OMNI
679682
elif model_task in ["sentence-similarity", "feature-extraction"]:
680683
effective_model_server = ModelServer.TEI
681684
else:
@@ -694,6 +697,36 @@ def _detect_huggingface_image(self) -> None:
694697
)
695698
self.framework = Framework.HUGGINGFACE
696699

700+
elif effective_model_server == ModelServer.VLLM:
701+
# vLLM: Use image_uris.retrieve with "huggingface-vllm" framework
702+
self.image_uri = image_uris.retrieve(
703+
"huggingface-vllm",
704+
region=self.region,
705+
version=None, # Use latest version
706+
image_scope="inference",
707+
)
708+
self.framework = Framework.HUGGINGFACE
709+
710+
elif effective_model_server == ModelServer.SGLANG:
711+
# SGLang: Use image_uris.retrieve with "huggingface-sglang" framework
712+
self.image_uri = image_uris.retrieve(
713+
"huggingface-sglang",
714+
region=self.region,
715+
version=None, # Use latest version
716+
image_scope="inference",
717+
)
718+
self.framework = Framework.HUGGINGFACE
719+
720+
elif effective_model_server == ModelServer.VLLM_OMNI:
721+
# vLLM-omni: Use image_uris.retrieve with "huggingface-vllm-omni" framework
722+
self.image_uri = image_uris.retrieve(
723+
"huggingface-vllm-omni",
724+
region=self.region,
725+
version=None, # Use latest version
726+
image_scope="inference",
727+
)
728+
self.framework = Framework.HUGGINGFACE
729+
697730
elif effective_model_server == ModelServer.TEI:
698731
# TEI: Use image_uris.retrieve with "huggingface-tei" framework
699732
self.image_uri = image_uris.retrieve(

‎sagemaker-serve/src/sagemaker/serve/utils/types.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ def __str__(self):
2020
TGI = 6
2121
TEI = 7
2222
SMD = 8
23+
VLLM = 9
24+
SGLANG = 10
25+
VLLM_OMNI = 11
2326

2427

2528
class HardwareType(Enum):

‎sagemaker-serve/tests/unit/test_model_builder_utils_extended_coverage.py‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,63 @@ def test_detect_hf_image_tei(self, mock_metadata, mock_retrieve):
150150
mock_retrieve.return_value = "763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-tei:latest"
151151

152152
utils._detect_huggingface_image()
153-
153+
154154
self.assertIsNotNone(utils.image_uri)
155155

156+
@patch('sagemaker.core.image_uris.retrieve')
157+
@patch.object(_ModelBuilderUtils, 'get_huggingface_model_metadata')
158+
def test_detect_hf_image_vllm(self, mock_metadata, mock_retrieve):
159+
"""Test HF image detection resolves the huggingface-vllm framework for vLLM."""
160+
utils = _ModelBuilderUtils()
161+
utils.model = "gpt2"
162+
utils.region = "us-west-2"
163+
utils.model_server = ModelServer.VLLM
164+
mock_retrieve.return_value = (
165+
"763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm:latest"
166+
)
167+
168+
utils._detect_huggingface_image()
169+
170+
self.assertIsNotNone(utils.image_uri)
171+
self.assertEqual(utils.framework, Framework.HUGGINGFACE)
172+
self.assertEqual(mock_retrieve.call_args.args[0], "huggingface-vllm")
173+
174+
@patch('sagemaker.core.image_uris.retrieve')
175+
@patch.object(_ModelBuilderUtils, 'get_huggingface_model_metadata')
176+
def test_detect_hf_image_sglang(self, mock_metadata, mock_retrieve):
177+
"""Test HF image detection resolves the huggingface-sglang framework for SGLang."""
178+
utils = _ModelBuilderUtils()
179+
utils.model = "gpt2"
180+
utils.region = "us-west-2"
181+
utils.model_server = ModelServer.SGLANG
182+
mock_retrieve.return_value = (
183+
"763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-sglang:latest"
184+
)
185+
186+
utils._detect_huggingface_image()
187+
188+
self.assertIsNotNone(utils.image_uri)
189+
self.assertEqual(utils.framework, Framework.HUGGINGFACE)
190+
self.assertEqual(mock_retrieve.call_args.args[0], "huggingface-sglang")
191+
192+
@patch('sagemaker.core.image_uris.retrieve')
193+
@patch.object(_ModelBuilderUtils, 'get_huggingface_model_metadata')
194+
def test_detect_hf_image_vllm_omni(self, mock_metadata, mock_retrieve):
195+
"""Test HF image detection resolves the huggingface-vllm-omni framework for omni."""
196+
utils = _ModelBuilderUtils()
197+
utils.model = "llava-hf/llava-1.5-7b-hf"
198+
utils.region = "us-west-2"
199+
utils.model_server = ModelServer.VLLM_OMNI
200+
mock_retrieve.return_value = (
201+
"763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm-omni:latest"
202+
)
203+
204+
utils._detect_huggingface_image()
205+
206+
self.assertIsNotNone(utils.image_uri)
207+
self.assertEqual(utils.framework, Framework.HUGGINGFACE)
208+
self.assertEqual(mock_retrieve.call_args.args[0], "huggingface-vllm-omni")
209+
156210

157211
class TestNormalizeFrameworkToEnum(unittest.TestCase):
158212
"""Test _normalize_framework_to_enum method."""

‎sagemaker-serve/tests/unit/test_model_builder_v3.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -938,37 +938,37 @@ def test_build_with_djl(
938938
self.assertEqual(result, mock_model)
939939
mock_build_djl.assert_called_once()
940940

941-
@patch('sagemaker.serve.model_builder.ModelBuilder._build_for_tgi')
941+
@patch('sagemaker.serve.model_builder.ModelBuilder._build_for_vllm')
942942
@patch('sagemaker.serve.model_builder.ModelBuilder._is_huggingface_model')
943943
@patch('sagemaker.serve.model_builder.ModelBuilder.get_huggingface_model_metadata')
944944
@patch('sagemaker.serve.model_builder.ModelBuilder._get_client_translators')
945945
@patch('sagemaker.serve.model_builder.ModelBuilder._get_serve_setting')
946946
@patch('sagemaker.serve.model_builder.ModelBuilder._build_validations')
947947
@patch('sagemaker.serve.model_builder.ModelBuilder._handle_mlflow_input')
948948
@patch('sagemaker.serve.model_builder.ModelBuilder._hf_schema_builder_init')
949-
def test_build_with_tgi_for_text_generation(
949+
def test_build_with_vllm_for_text_generation(
950950
self, mock_hf_schema_init, mock_mlflow, mock_validations, mock_serve_setting,
951-
mock_translators, mock_hf_metadata, mock_is_hf, mock_build_tgi
951+
mock_translators, mock_hf_metadata, mock_is_hf, mock_build_vllm
952952
):
953-
"""Test build with TGI for text-generation models."""
953+
"""Test build defaults to vLLM for text-generation models."""
954954
mock_model = Mock(spec=Model)
955-
mock_build_tgi.return_value = mock_model
955+
mock_build_vllm.return_value = mock_model
956956
mock_translators.return_value = (Mock(), Mock())
957957
mock_serve_setting.return_value = Mock()
958958
mock_is_hf.return_value = True
959959
mock_hf_metadata.return_value = {"pipeline_tag": "text-generation"}
960960
mock_hf_schema_init.return_value = None # Skip schema initialization
961-
961+
962962
builder = ModelBuilder(
963963
model="gpt2",
964964
role_arn=self.mock_role_arn,
965965
sagemaker_session=self.mock_session
966966
)
967-
967+
968968
result = builder._build_single_modelbuilder()
969-
969+
970970
self.assertEqual(result, mock_model)
971-
mock_build_tgi.assert_called_once()
971+
mock_build_vllm.assert_called_once()
972972

973973

974974
if __name__ == '__main__':

0 commit comments

Comments
 (0)