Skip to content

Commit 2c28d8f

Browse files
committed
fix(serve): pre-deploy JumpStart benchmark data + public HuggingFace download helper
Fixes two issues with ModelBuilder benchmark APIs for pre-deploy JumpStart models, plus adds a public HuggingFace-Hub download helper: - benchmark_metrics: display_benchmark_metrics() read self.benchmark_metrics, a property that existed in v2 but was dropped in the v3 port, so it raised AttributeError. Restore the property (a pandas DataFrame built from the model's published deployment-config benchmark data). - list_deployment_configs(): with no instance_type it called _get_deployment_configs() without first loading the JumpStart metadata configs, so a pre-deploy builder (_metadata_configs=None) got []. Move _ensure_metadata_configs() into _get_deployment_configs so every caller (listing and benchmark data) lazily loads them. - download_huggingface_model(): new public helper in serve.utils.hf_utils, exported from sagemaker.serve. Downloads a Hub snapshot and optionally uploads it to S3, so notebooks no longer hand-roll snapshot_download + S3Uploader. The previous downloader was a method on a private class and not importable. Unit: 9 new tests (4 benchmark/config, 5 HF helper).
1 parent d768dce commit 2c28d8f

6 files changed

Lines changed: 227 additions & 2 deletions

File tree

‎sagemaker-serve/src/sagemaker/serve/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
WorkloadValidationError,
4242
start_benchmark,
4343
)
44+
from sagemaker.serve.utils.hf_utils import download_huggingface_model
4445

4546
__all__ = [
4647
"InferenceSpec",
@@ -56,4 +57,5 @@
5657
"FeatureGatedError",
5758
"WorkloadValidationError",
5859
"start_benchmark",
60+
"download_huggingface_model",
5961
]

‎sagemaker-serve/src/sagemaker/serve/model_builder.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4782,6 +4782,20 @@ def transformer(
47824782
sagemaker_session=self.sagemaker_session,
47834783
)
47844784

4785+
@property
4786+
def benchmark_metrics(self):
4787+
"""Benchmark metrics for the model's JumpStart deployment configs.
4788+
4789+
Returns a pandas ``DataFrame`` (one row per config/instance) built from
4790+
the model's published benchmark data. Available for JumpStart models
4791+
(or HuggingFace models with a JumpStart equivalent) before deploy.
4792+
"""
4793+
import pandas as pd
4794+
4795+
df = pd.DataFrame(self._get_deployment_configs_benchmarks_data())
4796+
df.index = [""] * len(df)
4797+
return df
4798+
47854799
@_telemetry_emitter(
47864800
feature=Feature.MODEL_CUSTOMIZATION, func_name="model_builder.display_benchmark_metrics"
47874801
)

‎sagemaker-serve/src/sagemaker/serve/model_builder_utils.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2709,6 +2709,11 @@ def _get_deployment_configs(
27092709
selected_config_name (Optional[str]): The name of the selected deployment config.
27102710
selected_instance_type (Optional[str]): The selected instance type.
27112711
"""
2712+
# Lazily load the JumpStart metadata configs. Without this a pre-deploy
2713+
# builder (model set, build()/deploy() not yet called) has
2714+
# _metadata_configs=None, so both list_deployment_configs() and the
2715+
# benchmark-metrics data would come back empty.
2716+
self._ensure_metadata_configs()
27122717
deployment_configs = []
27132718
if not self._metadata_configs:
27142719
return deployment_configs

‎sagemaker-serve/src/sagemaker/serve/utils/hf_utils.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,82 @@
1414

1515
from __future__ import absolute_import
1616
import json
17+
import os
18+
import tempfile
1719
import urllib.request
1820
from json import JSONDecodeError
21+
from typing import Optional
1922
from urllib.error import HTTPError, URLError
2023
import logging
2124

2225
logger = logging.getLogger(__name__)
2326

2427

28+
def download_huggingface_model(
29+
model_id: str,
30+
*,
31+
local_dir: Optional[str] = None,
32+
s3_uri: Optional[str] = None,
33+
hf_hub_token: Optional[str] = None,
34+
sagemaker_session=None,
35+
) -> str:
36+
"""Download a HuggingFace Hub model snapshot, optionally staging it to S3.
37+
38+
A supported, importable helper so notebooks and scripts don't hand-roll
39+
``huggingface_hub.snapshot_download`` + ``S3Uploader``. Downloads the full
40+
model snapshot from the Hub, then either leaves it on local disk or uploads
41+
it to S3 and returns the resulting location.
42+
43+
Args:
44+
model_id: The HuggingFace Hub model id (e.g. ``"gpt2"``).
45+
local_dir: Local directory to download into. When ``s3_uri`` is given
46+
and ``local_dir`` is omitted, a temporary directory is used as a
47+
staging area for the upload. Defaults to ``None``.
48+
s3_uri: Optional ``s3://bucket/prefix`` destination. When set, the
49+
snapshot is uploaded there and the returned value is the S3 URI.
50+
hf_hub_token: Optional HuggingFace Hub token for gated/private models.
51+
sagemaker_session: Optional session used for the S3 upload. Defaults to
52+
a new session built from the ambient AWS configuration.
53+
54+
Returns:
55+
The S3 URI the snapshot was uploaded to when ``s3_uri`` is given,
56+
otherwise the local directory path the snapshot was downloaded into.
57+
58+
Raises:
59+
ImportError: If ``huggingface_hub`` is not installed.
60+
"""
61+
# Without an explicit local_dir, download to a temp dir only as a staging
62+
# area for the S3 upload; if neither local_dir nor s3_uri is given the
63+
# snapshot has nowhere to persist, which is a caller error.
64+
if local_dir is None and s3_uri is None:
65+
raise ValueError("Provide local_dir, s3_uri, or both.")
66+
67+
try:
68+
from huggingface_hub import snapshot_download
69+
except ImportError as exc:
70+
raise ImportError(
71+
"download_huggingface_model requires huggingface_hub, which is not "
72+
"installed. Install it with `pip install huggingface_hub`."
73+
) from exc
74+
75+
download_dir = local_dir or tempfile.mkdtemp(prefix="hf-model-")
76+
os.makedirs(download_dir, exist_ok=True)
77+
logger.info("Downloading model %s from Hugging Face Hub to %s", model_id, download_dir)
78+
snapshot_download(repo_id=model_id, local_dir=download_dir, token=hf_hub_token)
79+
80+
if s3_uri is None:
81+
return download_dir
82+
83+
from sagemaker.core.s3 import S3Uploader
84+
85+
logger.info("Uploading model %s snapshot to %s", model_id, s3_uri)
86+
return S3Uploader.upload(
87+
local_path=download_dir,
88+
desired_s3_uri=s3_uri,
89+
sagemaker_session=sagemaker_session,
90+
)
91+
92+
2593
def _get_model_config_properties_from_hf(model_id: str, hf_hub_token: str = None):
2694
"""Placeholder docstring"""
2795

‎sagemaker-serve/tests/unit/test_model_builder_coverage_boost.py‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"""
55

66
import unittest
7-
from unittest.mock import Mock, patch, MagicMock
7+
from unittest.mock import Mock, patch, MagicMock, PropertyMock
88
from dataclasses import dataclass
99
import tempfile
1010

@@ -333,6 +333,85 @@ def test_list_deployment_configs_non_string_model(self):
333333
self.assertIn("only supported for JumpStart", str(context.exception))
334334

335335

336+
class TestPreDeployBenchmarkData(unittest.TestCase):
337+
"""Pre-deploy JumpStart benchmark data: list_deployment_configs() and the
338+
benchmark_metrics property must work before build()/deploy() is called.
339+
"""
340+
341+
def _jumpstart_mb(self):
342+
"""A JumpStart-config ModelBuilder, constructed the same way as
343+
TestFromJumpStartConfig (explicit role_arn, no live role/instance
344+
resolution)."""
345+
js_config = JumpStartConfig(model_id="test-model", model_version="1.0.0")
346+
return ModelBuilder.from_jumpstart_config(
347+
jumpstart_config=js_config,
348+
role_arn="arn:aws:iam::123456789012:role/SageMakerRole",
349+
)
350+
351+
def test_get_deployment_configs_ensures_metadata(self):
352+
"""_get_deployment_configs lazily loads metadata configs itself, so every
353+
caller benefits. Pre-fix it read _metadata_configs (None pre-deploy) and
354+
returned [] without ever loading them."""
355+
mb = self._jumpstart_mb()
356+
mb._metadata_configs = None
357+
358+
with patch.object(mb, "_ensure_metadata_configs") as ensure:
359+
result = mb._get_deployment_configs(None, None)
360+
361+
ensure.assert_called_once()
362+
self.assertEqual(result, [])
363+
364+
def test_list_deployment_configs_loads_metadata_when_pre_deploy(self):
365+
"""list_deployment_configs() with no instance_type returns configs for a
366+
pre-deploy JumpStart model instead of []."""
367+
mb = self._jumpstart_mb()
368+
mb.config_name = None
369+
mb.instance_type = None
370+
371+
with patch.object(mb, "_is_jumpstart_model_id", return_value=True), patch.object(
372+
mb, "_is_model_customization", return_value=False
373+
), patch.object(mb, "_use_jumpstart_equivalent", return_value=False), patch.object(
374+
mb, "_get_deployment_configs", return_value=[Mock()]
375+
), patch.object(
376+
mb, "deployment_config_response_data", return_value=[{"DeploymentConfigName": "c1"}]
377+
):
378+
result = mb.list_deployment_configs()
379+
380+
self.assertEqual(result, [{"DeploymentConfigName": "c1"}])
381+
382+
def test_benchmark_metrics_property_returns_dataframe(self):
383+
"""The benchmark_metrics property builds a DataFrame from the config
384+
benchmark data (it did not exist before, causing AttributeError)."""
385+
mb = self._jumpstart_mb()
386+
sample = {
387+
"Instance Type": ["ml.g5.2xlarge", "ml.g5.12xlarge"],
388+
"Latency (ms)": [100.0, 80.0],
389+
}
390+
391+
with patch.object(mb, "_get_deployment_configs_benchmarks_data", return_value=sample):
392+
df = mb.benchmark_metrics
393+
394+
self.assertEqual(list(df["Instance Type"]), ["ml.g5.2xlarge", "ml.g5.12xlarge"])
395+
396+
def test_display_benchmark_metrics_no_attribute_error(self):
397+
"""display_benchmark_metrics() reads the benchmark_metrics property and
398+
no longer raises AttributeError for a JumpStart model. The property is
399+
patched to a stand-in frame so the assertion is on the wiring, not on
400+
pandas' optional markdown renderer."""
401+
mb = self._jumpstart_mb()
402+
df = MagicMock()
403+
df.to_markdown.return_value = "table"
404+
405+
with patch.object(mb, "_is_jumpstart_model_id", return_value=True), patch.object(
406+
mb, "_use_jumpstart_equivalent", return_value=False
407+
), patch.object(
408+
type(mb), "benchmark_metrics", new_callable=PropertyMock, return_value=df
409+
):
410+
mb.display_benchmark_metrics()
411+
412+
df.to_markdown.assert_called_once()
413+
414+
336415
class TestTransformer(unittest.TestCase):
337416
"""Test transformer method."""
338417

‎sagemaker-serve/tests/unit/utils/test_hf_utils.py‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
"""Unit tests for sagemaker.serve.utils.hf_utils module."""
22
import unittest
3+
import sys
34
from unittest.mock import Mock, patch, mock_open
45
import json
56
from urllib.error import HTTPError, URLError
67
from json import JSONDecodeError
7-
from sagemaker.serve.utils.hf_utils import _get_model_config_properties_from_hf
8+
from sagemaker.serve.utils.hf_utils import (
9+
_get_model_config_properties_from_hf,
10+
download_huggingface_model,
11+
)
812

913

1014
class TestGetModelConfigPropertiesFromHf(unittest.TestCase):
@@ -216,5 +220,58 @@ def _urlopen_side_effect(request):
216220
self.assertEqual(result, adapter_config)
217221

218222

223+
class TestDownloadHuggingfaceModel(unittest.TestCase):
224+
"""Test cases for the public download_huggingface_model helper."""
225+
226+
def setUp(self):
227+
# download_huggingface_model imports huggingface_hub lazily; provide a
228+
# stub module so the tests do not require the real package installed.
229+
self._hf_stub = Mock()
230+
self._patcher = patch.dict(sys.modules, {"huggingface_hub": self._hf_stub})
231+
self._patcher.start()
232+
self.addCleanup(self._patcher.stop)
233+
234+
def test_requires_local_dir_or_s3_uri(self):
235+
"""Neither destination given is a caller error, before any download."""
236+
with self.assertRaises(ValueError) as context:
237+
download_huggingface_model("gpt2")
238+
self.assertIn("local_dir, s3_uri", str(context.exception))
239+
self._hf_stub.snapshot_download.assert_not_called()
240+
241+
def test_downloads_to_local_dir_and_returns_it(self):
242+
"""With only local_dir, it downloads there and returns the path."""
243+
result = download_huggingface_model("gpt2", local_dir="/tmp/gpt2")
244+
self.assertEqual(result, "/tmp/gpt2")
245+
self._hf_stub.snapshot_download.assert_called_once_with(
246+
repo_id="gpt2", local_dir="/tmp/gpt2", token=None
247+
)
248+
249+
def test_uploads_to_s3_and_returns_uri(self):
250+
"""With s3_uri, it uploads the snapshot and returns the S3 URI."""
251+
with patch("sagemaker.core.s3.S3Uploader") as mock_uploader:
252+
mock_uploader.upload.return_value = "s3://bucket/prefix/gpt2"
253+
result = download_huggingface_model(
254+
"gpt2", local_dir="/tmp/gpt2", s3_uri="s3://bucket/prefix"
255+
)
256+
self.assertEqual(result, "s3://bucket/prefix/gpt2")
257+
mock_uploader.upload.assert_called_once()
258+
self.assertEqual(
259+
mock_uploader.upload.call_args.kwargs["desired_s3_uri"], "s3://bucket/prefix"
260+
)
261+
262+
def test_forwards_hf_token(self):
263+
"""A gated-model token is passed through to snapshot_download."""
264+
download_huggingface_model("gpt2", local_dir="/tmp/gpt2", hf_hub_token="hf_tok")
265+
self.assertEqual(self._hf_stub.snapshot_download.call_args.kwargs["token"], "hf_tok")
266+
267+
def test_missing_huggingface_hub_raises_import_error(self):
268+
"""A clear ImportError is raised when huggingface_hub is unavailable."""
269+
with patch.dict(sys.modules, {"huggingface_hub": None}), self.assertRaises(
270+
ImportError
271+
) as context:
272+
download_huggingface_model("gpt2", local_dir="/tmp/gpt2")
273+
self.assertIn("huggingface_hub", str(context.exception))
274+
275+
219276
if __name__ == "__main__":
220277
unittest.main()

0 commit comments

Comments
 (0)