|
16 | 16 | and deploying machine learning models across different model servers and deployment modes. |
17 | 17 | It supports various frameworks including PyTorch, TensorFlow, HuggingFace, XGBoost, and more. |
18 | 18 | """ |
| 19 | + |
19 | 20 | from __future__ import absolute_import, annotations |
20 | 21 |
|
21 | 22 | import os |
|
36 | 37 | _get_gpu_info_fallback, |
37 | 38 | ) |
38 | 39 | from sagemaker.serve.utils.hf_utils import _get_model_config_properties_from_hf |
39 | | -from sagemaker.serve.detector.image_detector import _get_model_base, _detect_framework_and_version |
| 40 | +from sagemaker.serve.detector.image_detector import ( |
| 41 | + _get_model_base, |
| 42 | + _detect_framework_and_version, |
| 43 | +) |
40 | 44 | from sagemaker.serve.detector.pickler import save_pkl |
41 | 45 | from sagemaker.serve.utils.types import ModelServer |
42 | 46 |
|
|
48 | 52 | from sagemaker.serve.model_server.tgi.utils import _get_default_tgi_configurations |
49 | 53 | from sagemaker.serve.model_server.tgi.prepare import prepare_tgi_js_resources |
50 | 54 | from sagemaker.serve.model_server.djl_serving.prepare import prepare_djl_js_resources |
51 | | -from sagemaker.serve.model_server.multi_model_server.prepare import prepare_mms_js_resources |
| 55 | +from sagemaker.serve.model_server.multi_model_server.prepare import ( |
| 56 | + prepare_mms_js_resources, |
| 57 | +) |
52 | 58 |
|
53 | 59 | # Model server preparation |
54 | 60 | from sagemaker.serve.model_server.torchserve.prepare import prepare_for_torchserve |
55 | | -from sagemaker.serve.model_server.tensorflow_serving.prepare import prepare_for_tf_serving |
| 61 | +from sagemaker.serve.model_server.tensorflow_serving.prepare import ( |
| 62 | + prepare_for_tf_serving, |
| 63 | +) |
56 | 64 | from sagemaker.serve.model_server.smd.prepare import prepare_for_smd |
57 | 65 | from sagemaker.serve.model_server.multi_model_server.prepare import prepare_for_mms |
58 | 66 |
|
|
70 | 78 |
|
71 | 79 |
|
72 | 80 | class _ModelBuilderServers(object): |
73 | | - |
74 | 81 | def _build_for_model_server(self) -> Model: |
75 | 82 | """Build model using explicit model server configuration. |
76 | 83 |
|
@@ -109,6 +116,12 @@ def _build_for_model_server(self) -> Model: |
109 | 116 | return self._build_for_tei() |
110 | 117 | elif self.model_server == ModelServer.TGI: |
111 | 118 | return self._build_for_tgi() |
| 119 | + elif self.model_server == ModelServer.VLLM: |
| 120 | + return self._build_for_vllm() |
| 121 | + elif self.model_server == ModelServer.SGLANG: |
| 122 | + return self._build_for_sglang() |
| 123 | + elif self.model_server == ModelServer.VLLM_OMNI: |
| 124 | + return self._build_for_vllm_omni() |
112 | 125 | elif self.model_server == ModelServer.MMS: |
113 | 126 | return self._build_for_transformers() |
114 | 127 | elif self.model_server == ModelServer.SMD: |
@@ -289,6 +302,97 @@ def _build_for_tgi(self) -> Model: |
289 | 302 |
|
290 | 303 | return model |
291 | 304 |
|
| 305 | + def _build_for_hf_server(self, model_server: ModelServer) -> Model: |
| 306 | + """Build a HuggingFace model for a hub-download serving container. |
| 307 | +
|
| 308 | + Generic build path shared by the vLLM, SGLang, and vLLM-omni servers. It |
| 309 | + configures the container to pull the model directly from the HuggingFace Hub |
| 310 | + (HF_MODEL_ID), resolves the appropriate DLC image via _auto_detect_image_uri, |
| 311 | + and prepares the model for the selected mode. Server-specific tuning (sharding, |
| 312 | + tensor parallelism, MAX_* limits, etc.) is intentionally left to the container |
| 313 | + defaults and will be added in a follow-up. |
| 314 | +
|
| 315 | + Args: |
| 316 | + model_server: The HuggingFace serving model server to build for |
| 317 | + (VLLM, SGLANG, or VLLM_OMNI). |
| 318 | +
|
| 319 | + Returns: |
| 320 | + Model: Configured model ready for deployment. |
| 321 | + """ |
| 322 | + self.secret_key = "" |
| 323 | + self.model_server = model_server |
| 324 | + |
| 325 | + # Use notebook instance type if available |
| 326 | + nb_instance = _get_nb_instance() |
| 327 | + if nb_instance and not self._user_provided_instance_type: |
| 328 | + self.instance_type = nb_instance |
| 329 | + logger.debug(f"Using detected notebook instance type: {nb_instance}") |
| 330 | + |
| 331 | + from sagemaker.serve.model_server.tgi.prepare import _create_dir_structure |
| 332 | + |
| 333 | + _create_dir_structure(self.model_path) |
| 334 | + |
| 335 | + if isinstance(self.model, str) and not self._is_jumpstart_model_id(): |
| 336 | + # These containers download the model directly from the HuggingFace Hub |
| 337 | + # Todo: missing something? |
| 338 | + self.env_vars.setdefault("HF_MODEL_ID", self.model) |
| 339 | + |
| 340 | + if self.env_vars.get("HUGGING_FACE_HUB_TOKEN"): |
| 341 | + self.env_vars["HF_TOKEN"] = self.env_vars.get("HUGGING_FACE_HUB_TOKEN") |
| 342 | + |
| 343 | + self.s3_upload_path = None |
| 344 | + |
| 345 | + self._auto_detect_image_uri() |
| 346 | + |
| 347 | + if not self._optimizing: |
| 348 | + if self.mode in LOCAL_MODES: |
| 349 | + self._prepare_for_mode(should_upload_artifacts=True) |
| 350 | + else: |
| 351 | + self.s3_model_data_url, _ = self._prepare_for_mode() |
| 352 | + |
| 353 | + # Cache management based on mode |
| 354 | + if self.mode in LOCAL_MODES: |
| 355 | + self.env_vars.update({"HF_HUB_OFFLINE": "1"}) |
| 356 | + else: |
| 357 | + self.env_vars["HF_HOME"] = "/tmp" |
| 358 | + self.env_vars["HUGGINGFACE_HUB_CACHE"] = "/tmp" |
| 359 | + |
| 360 | + if self.mode == Mode.SAGEMAKER_ENDPOINT and not self.instance_type: |
| 361 | + raise ValueError( |
| 362 | + "Instance type must be provided when building for SageMaker Endpoint mode." |
| 363 | + ) |
| 364 | + |
| 365 | + model = self._create_model() |
| 366 | + |
| 367 | + if "HF_HUB_OFFLINE" in self.env_vars: |
| 368 | + self.env_vars.update({"HF_HUB_OFFLINE": "0"}) |
| 369 | + |
| 370 | + return model |
| 371 | + |
| 372 | + def _build_for_vllm(self) -> Model: |
| 373 | + """Build a HuggingFace model for the vLLM serving container. |
| 374 | +
|
| 375 | + Returns: |
| 376 | + Model: Configured model ready for vLLM deployment. |
| 377 | + """ |
| 378 | + return self._build_for_hf_server(ModelServer.VLLM) |
| 379 | + |
| 380 | + def _build_for_sglang(self) -> Model: |
| 381 | + """Build a HuggingFace model for the SGLang serving container. |
| 382 | +
|
| 383 | + Returns: |
| 384 | + Model: Configured model ready for SGLang deployment. |
| 385 | + """ |
| 386 | + return self._build_for_hf_server(ModelServer.SGLANG) |
| 387 | + |
| 388 | + def _build_for_vllm_omni(self) -> Model: |
| 389 | + """Build a HuggingFace model for the vLLM-omni (multimodal) serving container. |
| 390 | +
|
| 391 | + Returns: |
| 392 | + Model: Configured model ready for vLLM-omni deployment. |
| 393 | + """ |
| 394 | + return self._build_for_hf_server(ModelServer.VLLM_OMNI) |
| 395 | + |
292 | 396 | def _build_for_djl(self) -> Model: |
293 | 397 | """Build model for DJL Serving deployment. |
294 | 398 |
|
|
0 commit comments