Harden LLM probe backend validation

Block ready decisions when explicit runtime evidence contradicts the
requested physics backend, preventing invalid benchmark measurements.

Author: Horde

tools/perf_bisection/docs/llm-policies.md

@@ -14,9 +14,9 @@ classification or binary-search decisions.
 No provider is selected by default. Always pass `--base_url`; remote endpoints
 must use HTTPS and URLs containing credentials are rejected. NVIDIA users must
 select an endpoint approved by the Agent Security Readiness portal (normally
-`inference.nvidia.com` when NVIDIA data or internal network access is present,
-or `build.nvidia.com` only in the documented isolated use case). Do not use a
-personal provider key with NVIDIA data.
+`inference-api.nvidia.com` when authorized for NVIDIA data or internal network
+access, or `build.nvidia.com` only in the documented isolated use case). Do not
+use a personal provider key with NVIDIA data.
 
 Deployments can enforce an exact hostname allowlist with the comma-separated
 `ISAACLAB_BISECTION_LLM_HOSTS` environment variable. Redirects are rejected so

tools/perf_bisection/src/isaaclab_bisection/bisection/probe.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import json
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Protocol
@@ -42,6 +43,10 @@
 )
 
 _PROMPT_PATH = Path(__file__).resolve().parents[1] / "prompts" / "container_probe.md"
+_RESOLVED_BACKEND_PATTERNS = (
+    re.compile(r"resolved physics backend:\s*([A-Za-z0-9_.-]+)", re.IGNORECASE),
+    re.compile(r"\bKit started,\s*backend\s*=\s*([A-Za-z0-9_.-]+)", re.IGNORECASE),
+)
 
 
 @dataclass(frozen=True)
@@ -121,6 +126,30 @@ def _context_prompt(ctx: ProbeContext) -> str:
     return redact_sensitive_text(json.dumps(payload, indent=2, sort_keys=True))
 
 
+def _backend_family(value: str) -> str | None:
+    """Return the supported physics-backend family named by ``value``."""
+    normalized = value.strip().lower()
+    if normalized.startswith("newton") or "newton_mjwarp" in normalized:
+        return "newton"
+    if normalized.startswith("physx") or "isaacsim_physx" in normalized:
+        return "physx"
+    return None
+
+
+def _explicit_backend_mismatch(ctx: ProbeContext) -> tuple[str, str] | None:
+    """Detect an explicit resolved physics backend that contradicts the plan."""
+    expected = _backend_family(ctx.backend_key)
+    if expected is None:
+        return None
+    evidence = "\n".join((ctx.live_output_tail, ctx.benchmark_log_tail))
+    for pattern in _RESOLVED_BACKEND_PATTERNS:
+        for match in pattern.finditer(evidence):
+            resolved = _backend_family(match.group(1))
+            if resolved is not None and resolved != expected:
+                return expected, resolved
+    return None
+
+
 @dataclass
 class NoProbePolicy:
     """Probe policy that immediately allows benchmarking."""
@@ -160,11 +189,22 @@ def decide(self, ctx: ProbeContext) -> ProbeDecision:
                 f"probe model unavailable: {exc}",
                 confidence="high",
             )
-        return self._parse(reply) or ProbeDecision(
-            PROBE_ACTION_HARNESS_BLOCKED,
-            "probe model returned an invalid decision",
-            confidence="high",
-        )
+        decision = self._parse(reply)
+        if decision is None:
+            return ProbeDecision(
+                PROBE_ACTION_HARNESS_BLOCKED,
+                "probe model returned an invalid decision",
+                confidence="high",
+            )
+        if decision.action == PROBE_ACTION_READY and (mismatch := _explicit_backend_mismatch(ctx)):
+            expected, resolved = mismatch
+            return ProbeDecision(
+                PROBE_ACTION_HARNESS_BLOCKED,
+                f"probe model marked the container ready despite an explicit backend mismatch: "
+                f"requested {expected}, resolved {resolved}",
+                confidence="high",
+            )
+        return decision
 
     def _parse(self, reply: str) -> ProbeDecision | None:
         text = reply.strip()

tools/perf_bisection/src/isaaclab_bisection/cli.py

@@ -233,7 +233,7 @@ def _parse_args() -> argparse.Namespace:
     local.add_argument(
         "--base_url",
         default=None,
-        help="OpenAI-compatible base URL for --recovery llm (defaults to the provider default).",
+        help="Explicit OpenAI-compatible base URL required for --recovery llm or --probe llm.",
     )
     local.add_argument(
         "--api_key_env",

tools/perf_bisection/src/isaaclab_bisection/prompts/container_probe.md

@@ -63,3 +63,10 @@ image. The orchestrator will reject packages outside `allowed_apt_packages`. For
 example, `Cannot find GMP` from a CMake build can be repaired with
 `"apt_packages": ["libgmp-dev"]` when that package is allowlisted. Use
 `harness_blocked` when no safe progress remains.
+
+Before returning `ready`, compare the requested physics backend with every
+explicitly resolved physics backend in the evidence. The backend families must
+match exactly. Renderer names do not override the resolved physics backend. For
+example, a plan requesting Newton with `resolved physics backend: physx` is a
+`plan_issue`, even when installation succeeded or a renderer name contains
+`newton`.

tools/perf_bisection/tests/test_bisect_runner.py

@@ -39,6 +39,7 @@
 from isaaclab_bisection.bisection.models import BisectionPlan, MetricSpec, RunnerSpec, TaskSpec  # noqa: E402
 from isaaclab_bisection.bisection.paired_reference import summarize_measurements  # noqa: E402
 from isaaclab_bisection.bisection.probe import (  # noqa: E402
+    PROBE_ACTION_HARNESS_BLOCKED,
     PROBE_ACTION_PLAN_ISSUE,
     PROBE_ACTION_READY,
     PROBE_ACTION_REPAIR_BASE_IMAGE,
@@ -1276,6 +1277,16 @@ def test_write_repair_dockerfile(self, tmp_path: Path) -> None:
 class TestLLMProbePolicy:
     """The LLM-driven probe parser accepts structured setup-doctor decisions."""
 
+    def _context(self, *, backend_key: str, live_output_tail: str) -> ProbeContext:
+        return ProbeContext(
+            commit_sha="0" * 40,
+            task_id="Isaac-Cartpole-Direct",
+            backend_key=backend_key,
+            artifact_dir=Path("."),
+            plan={"task_id": "Isaac-Cartpole-Direct", "backend_key": backend_key},
+            live_output_tail=live_output_tail,
+        )
+
     def test_parse_plan_issue_decision(self) -> None:
         policy = LLMProbePolicy(model="dummy-model")
         decision = policy._parse(
@@ -1316,6 +1327,42 @@ def test_parse_rejects_unknown_action(self) -> None:
         policy = LLMProbePolicy(model="dummy-model")
         assert policy._parse('{"action": "good", "reason": "not allowed"}') is None
 
+    def test_ready_is_blocked_when_explicit_backend_mismatches(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        policy = LLMProbePolicy(model="dummy-model")
+        monkeypatch.setattr(
+            policy._client,
+            "complete",
+            lambda system, user: '{"action":"ready","reason":"install succeeded","confidence":"high"}',
+        )
+
+        decision = policy.decide(
+            self._context(
+                backend_key="newton_newton_renderer",
+                live_output_tail="[INFO] resolved physics backend: physx\n"
+                "[INFO] renderer preset: physx_newton_renderer\n",
+            )
+        )
+
+        assert decision.action == PROBE_ACTION_HARNESS_BLOCKED
+        assert "requested newton, resolved physx" in decision.reason
+
+    def test_ready_is_allowed_when_explicit_backend_matches(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        policy = LLMProbePolicy(model="dummy-model")
+        monkeypatch.setattr(
+            policy._client,
+            "complete",
+            lambda system, user: '{"action":"ready","reason":"backend matches","confidence":"high"}',
+        )
+
+        decision = policy.decide(
+            self._context(
+                backend_key="newton",
+                live_output_tail="[INFO] Kit started, backend=newton\n",
+            )
+        )
+
+        assert decision.action == PROBE_ACTION_READY
+
 
 @dataclass
 class _FakeComponentStack:

tools/perf_bisection/tests/test_security.py

@@ -71,14 +71,14 @@ def test_llm_endpoint_must_be_explicit_and_secure() -> None:
     with pytest.raises(ValueError, match="private, link-local, or reserved"):
         validate_llm_base_url("https://169.254.169.254/v1")
 
-    assert validate_llm_base_url("https://inference.nvidia.com/v1/") == "https://inference.nvidia.com/v1"
+    assert validate_llm_base_url("https://inference-api.nvidia.com/v1/") == "https://inference-api.nvidia.com/v1"
     assert validate_llm_base_url("http://localhost:8000/v1") == "http://localhost:8000/v1"
 
 
 def test_llm_endpoint_honors_deployment_host_allowlist(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setenv("ISAACLAB_BISECTION_LLM_HOSTS", "inference.nvidia.com")
+    monkeypatch.setenv("ISAACLAB_BISECTION_LLM_HOSTS", "inference-api.nvidia.com")
 
-    assert validate_llm_base_url("https://inference.nvidia.com/v1") == "https://inference.nvidia.com/v1"
+    assert validate_llm_base_url("https://inference-api.nvidia.com/v1") == "https://inference-api.nvidia.com/v1"
     with pytest.raises(ValueError, match="not in ISAACLAB_BISECTION_LLM_HOSTS"):
         validate_llm_base_url("https://models.example.com/v1")