feat(serve): add SageMaker GenAI inference benchmarking and recommendation

Author: ZealSV

sagemaker-serve/VERSION

@@ -1 +1 @@
-1.13.0
+1.13.1

sagemaker-serve/src/sagemaker/serve/ai_inference_recommender/_recommendation_view.py

@@ -197,7 +197,7 @@ def __repr__(self) -> str:
             f"copy_count_per_instance: {_safe_str(dc, 'copy_count_per_instance')}",
             f"image_uri:            {_safe_str(dc, 'image_uri')}",
             f"model_package:        {_safe_str(md, 'model_package_arn')}",
-            f"inference_spec_name:  {_safe_str(md, 'inference_specification_name')}",
+            f"recommendation_spec_name: {_safe_str(md, 'inference_specification_name')}",
         ]
 
         env_vars = getattr(dc, "environment_variables", None) if dc else None
@@ -250,3 +250,65 @@ def _safe_float(obj, attr):
         return float(value) if value is not None else None
     except (TypeError, ValueError):
         return None
+
+
+class _RecommendationsView(list):
+    """List of recommendation rows with a comparative ``__repr__`` and a ``.best`` shortcut."""
+
+    @property
+    def best(self):
+        """The top-ranked recommendation row."""
+        if not self:
+            raise IndexError(
+                "No recommendations available. The AIRecommendationJob may "
+                "still be running, or it may have failed before producing "
+                "recommendations."
+            )
+        return self[0]
+
+    def __repr__(self) -> str:
+        if not self:
+            return "Recommendations[0]  (no rows)"
+
+        rows = []
+        for view in self:
+            md = getattr(view.raw, "model_details", None)
+            dc = getattr(view.raw, "deployment_configuration", None)
+            ep = view.expected_performance
+            rows.append([
+                f"[{view._index}]",
+                _safe_str(md, "inference_specification_name"),
+                _safe_str(dc, "instance_type"),
+                _safe_str(dc, "instance_count"),
+                _fmt_number(_get_metric_stat(ep, "request_throughput", "avg")),
+                _fmt_number(_get_metric_stat(ep, "output_token_throughput", "avg")),
+                _fmt_number(_get_metric_stat(ep, "request_latency", "p50")),
+                _fmt_number(_get_metric_stat(ep, "request_latency", "p90")),
+                _fmt_number(_get_metric_stat(ep, "request_latency", "p99")),
+                _fmt_number(_get_metric_stat(ep, "time_to_first_token", "p50")),
+                _fmt_number(_get_metric_stat(ep, "inter_token_latency", "p50")),
+            ])
+
+        table = _format_table(
+            headers=[
+                "idx", "spec_name", "instance_type", "cnt",
+                "req/s", "tok/s",
+                "lat_p50", "lat_p90", "lat_p99",
+                "ttft_p50", "itl_p50",
+            ],
+            rows=rows,
+        )
+
+        return (
+            f"Recommendations[{len(self)}]  (.best = top row; index by [N] for full detail)\n"
+            f"{table}\n"
+            f"lat/ttft/itl in ms; req/s = requests/sec; tok/s = output tokens/sec"
+        )
+
+
+def _get_metric_stat(ep_view, metric_name: str, stat: str):
+    try:
+        metric = getattr(ep_view, metric_name)
+    except AttributeError:
+        return None
+    return getattr(metric, stat, None)

sagemaker-serve/src/sagemaker/serve/model_builder.py

@@ -4298,25 +4298,32 @@ def generate_deployment_recommendations(
         return job
 
     @property
-    def recommendations(self) -> List:
+    def recommendations(self):
         """Recommendation rows from the most recent ``generate_deployment_recommendations`` call.
 
-        Returns an empty list if no recommendation job has been run, or if
-        the job has not produced recommendations yet (e.g., still
-        ``InProgress``). Each row is a read-only view that pretty-prints when
-        ``repr()``'d; field access forwards to the underlying service shape
-        (e.g., ``rec.deployment_configuration.instance_type``). Deploy via
-        ``self.deploy(recommendation_index=N)``.
+        Returns a list-like view; ``repr()`` renders a comparative table across
+        rows, and ``.best`` is a shortcut for the top-ranked row. Each row
+        forwards attribute access to the underlying service shape (e.g.,
+        ``rec.deployment_configuration.instance_type``).
+
+        Deploy via ``self.deploy(recommendation_index=N)`` or
+        ``self.deploy(recommendation_spec_name="...")``.
+
+        Empty if no recommendation job has been run, or if the job has not
+        produced recommendations yet.
         """
         from sagemaker.serve.ai_inference_recommender._recommendation_view import (
             _RecommendationView,
+            _RecommendationsView,
         )
 
         job = getattr(self, "_recommendation_job", None)
         if job is None:
-            return []
+            return _RecommendationsView()
         rows = list(job.recommendations or [])
-        return [_RecommendationView(row, index=i) for i, row in enumerate(rows)]
+        return _RecommendationsView(
+            _RecommendationView(row, index=i) for i, row in enumerate(rows)
+        )
 
     @classmethod
     def from_recommendation_job(
@@ -4331,7 +4338,7 @@ def from_recommendation_job(
         process didn't run (e.g. a job kicked off in a notebook last week,
         or owned by a teammate). After construction, call
         :meth:`deploy` with ``recommendation_index=`` or
-        ``inference_spec_name=`` as you would after ``generate_deployment_recommendations``.
+        ``recommendation_spec_name=`` as you would after ``generate_deployment_recommendations``.
 
         Args:
             recommendation_job: Either the name of an existing
@@ -4364,7 +4371,7 @@ def _deploy_recommendation(
         self,
         *,
         recommendation_index: int,
-        inference_spec_name: Optional[str],
+        recommendation_spec_name: Optional[str],
         endpoint_name: Optional[str],
         model_name: Optional[str],
         endpoint_config_name: Optional[str],
@@ -4381,7 +4388,7 @@ def _deploy_recommendation(
         We approve the package (rec jobs publish it un-approved) and create
         Model + EndpointConfig + Endpoint from it.
 
-        Row selection: ``inference_spec_name`` wins over
+        Row selection: ``recommendation_spec_name`` wins over
         ``recommendation_index`` when both are given.
         """
         import time as _time
@@ -4412,11 +4419,11 @@ def _deploy_recommendation(
                 f"{'Job failed: ' + str(failure_reason) if status == 'Failed' else 'Call job.wait() before deploy.'}"
             )
 
-        if inference_spec_name is not None:
+        if recommendation_spec_name is not None:
             matches = [
                 row for row in rows
                 if getattr(getattr(row, "model_details", None), "inference_specification_name", None)
-                == inference_spec_name
+                == recommendation_spec_name
             ]
             if not matches:
                 available = sorted({
@@ -4427,8 +4434,8 @@ def _deploy_recommendation(
                     if name
                 })
                 raise ValueError(
-                    f"No recommendation row matches inference_spec_name="
-                    f"{inference_spec_name!r}. "
+                    f"No recommendation row matches recommendation_spec_name="
+                    f"{recommendation_spec_name!r}. "
                     f"Available: {available or '(none reported)'}."
                 )
             rec = matches[0]
@@ -4536,7 +4543,7 @@ def deploy(
         # was called previously, or when this builder was hydrated via
         # ModelBuilder.from_recommendation_job(...)).
         recommendation_index: int = 0,
-        inference_spec_name: Optional[str] = None,
+        recommendation_spec_name: Optional[str] = None,
         role: Optional[str] = None,
         model_name: Optional[str] = None,
         endpoint_config_name: Optional[str] = None,
@@ -4601,7 +4608,7 @@ def deploy(
         if getattr(self, "_recommendation_job", None) is not None:
             return self._deploy_recommendation(
                 recommendation_index=recommendation_index,
-                inference_spec_name=inference_spec_name,
+                recommendation_spec_name=recommendation_spec_name,
                 endpoint_name=endpoint_name,
                 model_name=model_name,
                 endpoint_config_name=endpoint_config_name,

sagemaker-serve/tests/integ/test_ai_inference_recommender_integration.py

@@ -173,10 +173,15 @@ def test_recommendation_workflow_end_to_end():
         assert rows, (
             "ModelBuilder.recommendations is empty after generate_deployment_recommendations"
         )
-        top = rows[0]
+        top = rows.best
+        assert top is rows[0], "rows.best should equal rows[0]"
         assert getattr(getattr(top, "model_details", None), "model_package_arn", None), (
             f"Top recommendation has no ModelPackageArn. Raw: {top}"
         )
+        # Comparative table __repr__ should include both row indices and column headers.
+        rec_table = repr(rows)
+        assert f"Recommendations[{len(rows)}]" in rec_table
+        assert "spec_name" in rec_table and "req/s" in rec_table
         logger.info("Got %d recommendations; deploying top one", len(rows))
 
         rec_endpoint = model_builder.deploy(

sagemaker-serve/tests/unit/test_ai_inference_recommender/test_jobs.py

@@ -24,6 +24,7 @@
 )
 from sagemaker.serve.ai_inference_recommender._recommendation_view import (
     _RecommendationView,
+    _RecommendationsView,
 )
 
 
@@ -128,6 +129,11 @@ def test_repr_renders_config_and_perf_table(self):
         assert "ml.g6.12xlarge" in text
         assert "model-package/p/1" in text
         assert "my-spec" in text
+        # The label uses the new "recommendation_spec_name" naming, not the
+        # legacy "inference_spec_name" (which collides with the model-package
+        # InferenceSpecification concept).
+        assert "recommendation_spec_name:" in text
+        assert "inference_spec_name:" not in text
         # Env vars
         assert "MY_VAR_A = 128" in text
         # Expected performance table
@@ -152,3 +158,90 @@ def test_handles_missing_optional_fields(self):
         assert "Recommendation[0]" in text
         assert "instance_type:        -" in text
         assert "model_package:        -" in text
+
+
+def _fake_two_recommendations():
+    """Two rows that differ enough that the comparative table can render them distinctly."""
+    row_a = SimpleNamespace(
+        model_details=SimpleNamespace(
+            model_package_arn="arn:.../p/A",
+            inference_specification_name="high-otps-on-g5-2xlarge",
+        ),
+        deployment_configuration=SimpleNamespace(
+            instance_type="ml.g5.2xlarge",
+            instance_count=1,
+            copy_count_per_instance=1,
+            image_uri="example/image:1",
+            environment_variables={},
+        ),
+        expected_performance=[
+            SimpleNamespace(metric="RequestThroughput", stat="avg", value=152.94, unit="Requests/Second"),
+            SimpleNamespace(metric="OutputTokenThroughput", stat="avg", value=4893.7, unit="Tokens/Second"),
+            SimpleNamespace(metric="RequestLatency", stat="p50", value=402.6, unit="Milliseconds"),
+            SimpleNamespace(metric="RequestLatency", stat="p99", value=481.6, unit="Milliseconds"),
+        ],
+    )
+    row_b = SimpleNamespace(
+        model_details=SimpleNamespace(
+            model_package_arn="arn:.../p/B",
+            inference_specification_name="high-otps-on-g5-2xlarge-1",
+        ),
+        deployment_configuration=SimpleNamespace(
+            instance_type="ml.g5.2xlarge",
+            instance_count=1,
+            copy_count_per_instance=1,
+            image_uri="example/image:2",
+            environment_variables={},
+        ),
+        expected_performance=[
+            SimpleNamespace(metric="RequestThroughput", stat="avg", value=151.6, unit="Requests/Second"),
+            SimpleNamespace(metric="OutputTokenThroughput", stat="avg", value=4851.1, unit="Tokens/Second"),
+            SimpleNamespace(metric="RequestLatency", stat="p50", value=425.2, unit="Milliseconds"),
+            SimpleNamespace(metric="RequestLatency", stat="p99", value=474.6, unit="Milliseconds"),
+        ],
+    )
+    return [row_a, row_b]
+
+
+class TestRecommendationsView:
+    def _make(self, rows=None):
+        rows = rows if rows is not None else _fake_two_recommendations()
+        return _RecommendationsView(
+            _RecommendationView(row, index=i) for i, row in enumerate(rows)
+        )
+
+    def test_behaves_like_list(self):
+        view = self._make()
+        assert len(view) == 2
+        assert isinstance(view[0], _RecommendationView)
+        # iteration
+        names = [getattr(getattr(r.raw, "model_details", None), "inference_specification_name", None) for r in view]
+        assert names == ["high-otps-on-g5-2xlarge", "high-otps-on-g5-2xlarge-1"]
+
+    def test_best_returns_first_row(self):
+        view = self._make()
+        assert view.best is view[0]
+
+    def test_best_on_empty_raises_indexerror_with_helpful_message(self):
+        view = _RecommendationsView()
+        with pytest.raises(IndexError, match="No recommendations available"):
+            _ = view.best
+
+    def test_repr_renders_comparative_table_with_both_rows(self):
+        text = repr(self._make())
+        # Header / framing
+        assert "Recommendations[2]" in text
+        # Both spec names land in the table
+        assert "high-otps-on-g5-2xlarge" in text
+        assert "high-otps-on-g5-2xlarge-1" in text
+        # Column headers
+        for column in ("idx", "spec_name", "instance_type", "req/s", "lat_p99"):
+            assert column in text
+        # The two rows have different throughput values that should both render
+        assert "152.9" in text or "152.94" in text
+        assert "151.6" in text or "151.60" in text
+
+    def test_repr_when_empty(self):
+        text = repr(_RecommendationsView())
+        # Should not crash and should clearly indicate emptiness
+        assert "Recommendations[0]" in text or "no rows" in text