Skip to content

fix(serve): handle partial base model metadata #6466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lucasjia-aws wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix(serve): handle partial base model metadata #6466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/model_customization/deploy_sagemaker_endpoint.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,31 @@ production deployments.
endpoint = model_builder.deploy(endpoint_name="my-endpoint")


Select a Deployment Recipe
----------------------------

``ModelBuilder`` normally reads the recipe name from the model package's
base-model metadata. If that metadata omits the recipe, it first uses top-level
``HostingConfigs`` from the Hub document. When no top-level configuration
exists, it automatically uses the only recipe that publishes
``HostingConfigs``. If multiple recipes are deployable, pass ``recipe_name``
explicitly:

.. code-block:: python

from sagemaker.serve import ModelBuilder

model_builder = ModelBuilder(
model=model_package,
recipe_name="my-deployment-recipe",
)
endpoint = model_builder.build().deploy(endpoint_name="my-endpoint")

The override is only needed to disambiguate recipe-level hosting
configurations; existing model packages that publish a recipe name or top-level
hosting configuration require no change.


Reuse Deployed Resources
--------------------------

Expand Down
6 changes: 6 additions & 0 deletions migration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,12 @@ predictor = model_builder.deploy(
)
```

For customized model packages, V3 reads the deployment recipe from the package's base-model
metadata. If the recipe is omitted, ``ModelBuilder`` prefers top-level Hub ``HostingConfigs`` and
otherwise infers the recipe only when exactly one recipe is deployable. Pass
``recipe_name="<recipe>"`` to ``ModelBuilder`` when multiple recipes publish hosting
configurations.

### 3. Processing Jobs

**V2 Processor:**
Expand Down
168 changes: 114 additions & 54 deletions sagemaker-serve/src/sagemaker/serve/model_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
ModelCard,
ModelPackageModelCard,
)
from sagemaker.core.utils.utils import logger
from sagemaker.core.utils.utils import Unassigned, logger
from sagemaker.core.exceptions import CapacityError, UnexpectedStatusException
from sagemaker.core.helper import session_helper
from sagemaker.core.helper.iam_role_resolver import resolve_and_validate_role
Expand Down Expand Up @@ -261,6 +261,8 @@ class ModelBuilder(_InferenceRecommenderMixin, _ModelBuilderServers, _ModelBuild
network: Network configuration including VPC settings and network isolation.
instance_type: EC2 instance type for deployment (e.g., 'ml.m5.large').
mode: Deployment mode (SAGEMAKER_ENDPOINT, LOCAL_CONTAINER, or IN_PROCESS).
recipe_name: Optional recipe name used to select recipe-level hosting configuration when
base model metadata omits it or the Hub document publishes multiple deployable recipes.

Note:
ModelBuilder returns sagemaker.core.resources.Model and sagemaker.core.resources.Endpoint
Expand Down Expand Up @@ -450,6 +452,14 @@ class ModelBuilder(_InferenceRecommenderMixin, _ModelBuilderServers, _ModelBuild
default=None,
metadata={"help": "DEPRECATED: Use configure_for_torchserve() instead"},
)
recipe_name: Optional[str] = field(
default=None,
metadata={
"help": "Recipe name override for resolving recipe-level hosting configurations. "
"Use this when base model metadata omits a recipe and the Hub document publishes "
"multiple recipes with hosting configurations."
},
)

def _create_session_with_region(self):
"""Create a SageMaker session with the correct region."""
Expand Down Expand Up @@ -750,22 +760,47 @@ def _infer_instance_type_from_jumpstart(self) -> str:

raise ValueError(error_msg)

@staticmethod
def _optional_base_model_string(value: Any) -> Optional[str]:
"""Normalize an optional string from generated ``BaseModel`` metadata."""
if value is None or isinstance(value, Unassigned) or value == "":
return None
return value

def _effective_recipe_name(self, base_model: Any) -> Optional[str]:

@github-actions github-actions Bot Oct 13, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Low: _effective_recipe_name prefers the public self.recipe_name override over the BaseModel's own recipe_name in every call site, including PEFT detection (_fetch_peft, _deploy_model_customization) and _is_nova_model, not just hosting-config disambiguation. The docstring/field help describes recipe_name as a disambiguation override, so a user who sets it to pick a hosting config would also silently change PEFT-type ("lora" in recipe_name) and Nova detection. Consider restricting the override to the hosting-config resolution paths, or documenting that it influences PEFT/Nova inference as well.

"""Return the public recipe override or the normalized BaseModel recipe."""
return self._optional_base_model_string(
self.recipe_name
) or self._optional_base_model_string(getattr(base_model, "recipe_name", None))

def _fetch_hub_document_for_custom_model(self) -> dict:
from sagemaker.core.shapes import BaseModel as CoreBaseModel

base_model: CoreBaseModel = (
self._fetch_model_package().inference_specification.containers[0].base_model
)
hub_content_name = self._optional_base_model_string(base_model.hub_content_name)
if hub_content_name is None:
raise ValueError(
"Cannot resolve hosting configuration because the model package's BaseModel "
"does not specify hub_content_name."
)
hub_content_version = self._optional_base_model_string(base_model.hub_content_version)

# Prefer an explicitly configured hub (e.g. a private hub used for
# testing or for sourcing pre-release hosting configs); fall back to the
# public JumpStart hub when none is set.
hub_name = getattr(self, "hub_name", None) or "SageMakerPublicHub"
hub_content = HubContent.get(
hub_content_type="Model",
hub_name=hub_name,
hub_content_name=base_model.hub_content_name,
hub_content_version=base_model.hub_content_version,
)
get_kwargs = {
"hub_content_type": "Model",
"hub_name": hub_name,
"hub_content_name": hub_content_name,
}
# Omitting HubContentVersion asks SageMaker for the latest version. Passing
# None, an empty string, or Unassigned would serialize an invalid request.
if hub_content_version is not None:
get_kwargs["hub_content_version"] = hub_content_version
hub_content = HubContent.get(**get_kwargs)
return json.loads(hub_content.hub_content_document)

def _fetch_hosting_configs_for_custom_model(self) -> dict:
Expand Down Expand Up @@ -1011,28 +1046,41 @@ def _is_gpu_instance(self, instance_type: str) -> bool:

@staticmethod
def _extract_hosting_configs_from_hub(
hub_document: Dict[str, Any], recipe_name: str
hub_document: Dict[str, Any], recipe_name: Optional[str]
) -> List[Dict[str, Any]]:
"""Discover the raw ``HostingConfigs`` for a recipe from an already-fetched hub document.

The authoritative hosting configurations live under
``RecipeCollection[<recipe>].HostingConfigs``. Falls back to the top-level
``HostingConfigs`` when the matching recipe entry does not carry its own (some models
publish there instead). Returns ``[]`` when neither is present — callers decide whether an
empty result is an error (fine-tuned) or a signal to try another path (e.g. Nova).

This is the single source of truth for config discovery so that
:meth:`_resolve_recipe_hosting_configs` (used by list/get/set_deployment_config) and
:meth:`_fetch_and_cache_recipe_config` (used by build) never diverge — in particular the
top-level fallback is honored on BOTH the selection and the build paths.
"""Discover raw ``HostingConfigs`` using explicit, top-level, or unique recipe metadata.

A named recipe is selected first and retains the historical top-level fallback. When no
recipe is available, top-level ``HostingConfigs`` takes precedence. Only if top-level
configs are absent does ModelBuilder infer a recipe, and only when exactly one recipe
publishes non-empty hosting configs.
"""
for recipe in hub_document.get("RecipeCollection", []):
if recipe.get("Name") == recipe_name:
configs = recipe.get("HostingConfigs")
if configs:
return configs
break
return hub_document.get("HostingConfigs") or []
recipe_name = ModelBuilder._optional_base_model_string(recipe_name)
recipes = hub_document.get("RecipeCollection", []) or []
if recipe_name is not None:
for recipe in recipes:
if recipe.get("Name") == recipe_name:
configs = recipe.get("HostingConfigs")
if configs:
return configs
break

top_level_configs = hub_document.get("HostingConfigs")
if top_level_configs:
return top_level_configs
if recipe_name is not None:
return []

candidates = [recipe for recipe in recipes if recipe.get("HostingConfigs")]
if len(candidates) == 1:
return candidates[0]["HostingConfigs"]
if len(candidates) > 1:
candidate_names = [str(recipe.get("Name") or "<unnamed>") for recipe in candidates]
raise ValueError(
"Multiple recipes publish HostingConfigs: "
f"{', '.join(candidate_names)}. Specify recipe_name when creating ModelBuilder."
)
return []

def _resolve_recipe_hosting_configs(self) -> List[Dict[str, Any]]:
"""Return the raw ``HostingConfigs`` list published for this model's recipe.
Expand All @@ -1052,7 +1100,7 @@ def _resolve_recipe_hosting_configs(self) -> List[Dict[str, Any]]:
"Model is not supported for deployment: the model package has no inference "
"container to resolve a recipe from."
)
recipe_name = getattr(containers[0].base_model, "recipe_name", None) or ""
recipe_name = self._effective_recipe_name(containers[0].base_model)

configs = self._extract_hosting_configs_from_hub(hub_document, recipe_name)
if configs:
Expand Down Expand Up @@ -1286,7 +1334,7 @@ def _fetch_and_cache_recipe_config(self):
hub_document = self._fetch_hub_document_for_custom_model()
model_package = self._fetch_model_package()
container = model_package.inference_specification.containers[0]
recipe_name = getattr(container.base_model, "recipe_name", None) or ""
recipe_name = self._effective_recipe_name(container.base_model)

if not self.s3_upload_path:
from sagemaker.serve.utils.model_package_utils import get_s3_uri_from_inference_spec
Expand Down Expand Up @@ -1546,7 +1594,11 @@ def _base_model_name(self) -> Optional[str]:
model_package = self._fetch_model_package()
if model_package is not None:
base_model = model_package.inference_specification.containers[0].base_model
return getattr(base_model, "hub_content_name", None) if base_model else None
return (
self._optional_base_model_string(getattr(base_model, "hub_content_name", None))
if base_model
else None
)
if isinstance(self.model, BaseTrainer):
return self.model.base_model_name
# Raw S3 checkpoint: base model identity is supplied via model_metadata.
Expand All @@ -1572,8 +1624,13 @@ def _is_nova_model(self) -> bool:
if containers:
base_model = getattr(containers[0], "base_model", None)
if base_model:
recipe_name = getattr(base_model, "recipe_name", None) or ""
hub_content_name = getattr(base_model, "hub_content_name", None) or ""
recipe_name = self._effective_recipe_name(base_model) or ""
hub_content_name = (
self._optional_base_model_string(
getattr(base_model, "hub_content_name", None)
)
or ""
)
if (isinstance(recipe_name, str) and "nova" in recipe_name.lower()) or (
isinstance(hub_content_name, str) and "nova" in hub_content_name.lower()
):
Expand Down Expand Up @@ -1645,16 +1702,9 @@ def _get_nova_hosting_config_from_hub_document(self, instance_type=None):
return None

container = self._fetch_model_package().inference_specification.containers[0]
recipe_name = getattr(container.base_model, "recipe_name", None) or ""

hosting_configs = None
for recipe in hub_document.get("RecipeCollection", []):
if recipe.get("Name") == recipe_name:
hosting_configs = recipe.get("HostingConfigs")
break
if not hosting_configs:
hosting_configs = hub_document.get("HostingConfigs")
recipe_name = self._effective_recipe_name(container.base_model)

hosting_configs = self._extract_hosting_configs_from_hub(hub_document, recipe_name)
if not hosting_configs:
return None

Expand Down Expand Up @@ -1690,9 +1740,9 @@ def _get_nova_hosting_config(self, instance_type=None):

model_package = self._fetch_model_package()
if model_package:
hub_content_name = model_package.inference_specification.containers[
0
].base_model.hub_content_name
hub_content_name = self._optional_base_model_string(
model_package.inference_specification.containers[0].base_model.hub_content_name
)
else:
# No model package (e.g. SMTJ trainer): resolve from base_model_name
base_model_name = self._base_model_name()
Expand Down Expand Up @@ -1747,7 +1797,9 @@ def _validate_nova_smi_config(self) -> None:
base_model = getattr(containers[0], "base_model", None)
if not base_model:
return
hub_content_name = getattr(base_model, "hub_content_name", None)
hub_content_name = self._optional_base_model_string(
getattr(base_model, "hub_content_name", None)
)
if not hub_content_name:
return

Expand Down Expand Up @@ -3286,7 +3338,13 @@ def fetch_endpoint_names_for_base_model(self) -> Set[str]:
model_package = self._fetch_model_package()
if is_restricted_model_package(model_package):
return set()
recipe_name = model_package.inference_specification.containers[0].base_model.recipe_name
base_model = model_package.inference_specification.containers[0].base_model
recipe_name = self._effective_recipe_name(base_model)
if recipe_name is None:
raise ValueError(

@github-actions github-actions Bot Oct 13, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 Medium: fetch_endpoint_names_for_base_model now raises ValueError when _effective_recipe_name returns None, which is new behavior (previously it read base_model.recipe_name directly). This is a reasonable change, but none of the nine added tests exercise this new raise path, and the method had existing unit coverage (test_model_builder.py::test_fetch_endpoint_names_for_base_model). Please add a test asserting the ValueError is raised when no recipe is resolvable so this contract is pinned.

"Cannot fetch endpoints for the base model because no recipe is available. "
"Specify recipe_name when creating ModelBuilder."
)
endpoint_names = set()
logger.error(f"recipe_name: {recipe_name}")
for inference_component in InferenceComponent.get_all():
Expand Down Expand Up @@ -6465,8 +6523,8 @@ def _deploy_model_customization(
base_model_recipe_name = None
if peft_type == "LORA":
container = model_package.inference_specification.containers[0]
base_model_recipe_name = getattr(
getattr(container, "base_model", None), "recipe_name", None
base_model_recipe_name = self._effective_recipe_name(
getattr(container, "base_model", None)
)
if not base_model_recipe_name:
raise ValueError(
Expand Down Expand Up @@ -6681,7 +6739,7 @@ def _fetch_peft(self) -> Optional[str]:
if getattr(container, "is_checkpoint", None) is False:
return None
recipe_name = (
getattr(getattr(container, "base_model", None), "recipe_name", "") or ""
self._effective_recipe_name(getattr(container, "base_model", None)) or ""
)
if "lora" in recipe_name.lower():
return "LORA"
Expand Down Expand Up @@ -6766,14 +6824,16 @@ def _deploy_nova_model(
# The jumpstart-model-id tag always applies (resolved from the model
# package or the trainer's base_model_name). The recipe-name tag is only
# available when a model package is present.
tags = [
{"key": "sagemaker-sdk:jumpstart-model-id", "value": self._base_model_name()},
]
tags = []
base_model_name = self._optional_base_model_string(self._base_model_name())
if base_model_name is not None:
tags.append({"key": "sagemaker-sdk:jumpstart-model-id", "value": base_model_name})
model_package = self._fetch_model_package()
if model_package is not None:
base_model = model_package.inference_specification.containers[0].base_model
if base_model is not None and base_model.recipe_name:
tags.append({"key": "sagemaker-sdk:recipe-name", "value": base_model.recipe_name})
recipe_name = self._effective_recipe_name(base_model)
if recipe_name is not None:
tags.append({"key": "sagemaker-sdk:recipe-name", "value": recipe_name})

# Merge tags accumulated via add_tags (e.g. the model-source reuse tag).
# Those are stored in {"Key": ..., "Value": ...} form, so normalize to the
Expand Down
Loading
Loading