add: huggingface-llamacpp routing

Signed-off-by: DWarez <dario.salvati@huggingface.co>

Author: dwarez

sagemaker-serve/src/sagemaker/serve/constants.py

@@ -68,6 +68,7 @@
     ModelServer.VLLM,
     ModelServer.SGLANG,
     ModelServer.VLLM_OMNI,
+    ModelServer.LLAMACPP,
 }
 
 # HuggingFace pipeline tags that should route to the vLLM-omni (multimodal) server.

sagemaker-serve/src/sagemaker/serve/model_builder_servers.py

@@ -122,6 +122,8 @@ def _build_for_model_server(self) -> Model:
             return self._build_for_sglang()
         elif self.model_server == ModelServer.VLLM_OMNI:
             return self._build_for_vllm_omni()
+        elif self.model_server == ModelServer.LLAMACPP:
+            return self._build_for_llamacpp()
         elif self.model_server == ModelServer.MMS:
             return self._build_for_transformers()
         elif self.model_server == ModelServer.SMD:
@@ -393,6 +395,14 @@ def _build_for_vllm_omni(self) -> Model:
         """
         return self._build_for_hf_server(ModelServer.VLLM_OMNI)
 
+    def _build_for_llamacpp(self) -> Model:
+        """Build a HuggingFace model for the llama.cpp serving container.
+
+        Returns:
+            Model: Configured model ready for llama.cpp deployment.
+        """
+        return self._build_for_hf_server(ModelServer.LLAMACPP)
+
     def _build_for_djl(self) -> Model:
         """Build model for DJL Serving deployment.
 

sagemaker-serve/src/sagemaker/serve/model_builder_utils.py

@@ -727,6 +727,18 @@ def _detect_huggingface_image(self) -> None:
                 )
                 self.framework = Framework.HUGGINGFACE
 
+            elif effective_model_server == ModelServer.LLAMACPP:
+                # llama.cpp: Use image_uris.retrieve with "huggingface-llamacpp" framework.
+                # This DLC ships both gpu and cpu variants, so instance_type is required to select the right processor.
+                self.image_uri = image_uris.retrieve(
+                    "huggingface-llamacpp",
+                    region=self.region,
+                    version=None,  # Use latest version
+                    image_scope="inference",
+                    instance_type=getattr(self, "instance_type", None),
+                )
+                self.framework = Framework.HUGGINGFACE
+
             elif effective_model_server == ModelServer.TEI:
                 # TEI: Use image_uris.retrieve with "huggingface-tei" framework
                 self.image_uri = image_uris.retrieve(

sagemaker-serve/src/sagemaker/serve/utils/types.py

@@ -23,6 +23,7 @@ def __str__(self):
     VLLM = 9
     SGLANG = 10
     VLLM_OMNI = 11
+    LLAMACPP = 12
 
 
 class HardwareType(Enum):