Scope stability qualification to the Newton buckets

The 2026-07-28 run collected clean evidence for the four PhysX-only
buckets and none for the five that load Newton, which all crashed on
`from newton.solvers import SolverNotifyFlags`. Re-running the full
matrix would spend half the pool time re-measuring buckets that already
worked.

Restrict both the sampling and the qualification to the Newton-touching
backends, cutting the run from 135 to 75 samples. The seeder already
accepted a backend allowlist; runner_stability now takes the same list
so the expected scope matches what was collected, instead of reporting
the PhysX buckets as missing evidence and failing closed.

An unknown backend key is rejected rather than silently dropped, since
a typo would otherwise narrow the scope and yield a weaker verdict that
still reads as qualified.

Author: Horde

.github/workflows/perf-smoke-runner-stability.yaml

@@ -8,6 +8,10 @@
 # the commit out to five independent allocations so per-runner and run-to-run FPS
 # spread can be measured separately. Qualification only reports a verdict; it
 # never writes baselines.
+#
+# Scope is the Newton-touching buckets. The PhysX-only buckets already produced
+# clean evidence on 2026-07-28; these five crashed on a Newton import and are the
+# ones still lacking a verdict. Halving the matrix also halves the pool time.
 
 name: Performance Smoke - L40S Runner Stability
 
@@ -29,6 +33,11 @@ concurrency:
   group: perf-smoke-runner-stability-${{ github.ref }}
   cancel-in-progress: false
 
+env:
+  # Every backend_key whose benchmark loads Newton. Covers five task/backend
+  # buckets, since `newton` applies to both Cartpole and Velocity-Flat-G1.
+  STABILITY_BACKENDS: "newton,newton_rtx_renderer,newton_newton_renderer,physx_newton_renderer"
+
 jobs:
   wait_for_quiet_pool:
     name: Wait for initial gate and quiet pool
@@ -102,6 +111,9 @@ jobs:
       commit_count: "1"
       samples_per_commit: "3"
       tasks: "__ALL_TASKS__"
+      # Literal because `with:` cannot read the `env` context. Kept in sync with
+      # STABILITY_BACKENDS by test_staging_workflow_qualifies_only_newton_buckets.
+      backends: "newton,newton_rtx_renderer,newton_newton_renderer,physx_newton_renderer"
       target_branch: perf-smoke/develop-staging
       strict_ancestry: true
       dry_run: true
@@ -156,6 +168,7 @@ jobs:
         done
         python3 tools/perf_smoke_test/runner_stability.py \
           "${RECORD_ARGS[@]}" \
+          --backends "${STABILITY_BACKENDS}" \
           --gpu_model l40s \
           --expected_target_branch perf-smoke/develop-staging \
           --expected_commit "${GITHUB_SHA}" \

tools/perf_smoke_test/runner_stability.py

@@ -124,6 +124,30 @@ def _noise_floor_for_task(task: TaskConfig, gpu_model: str) -> float:
     return 0.0
 
 
+def select_backends(tasks: list[TaskConfig], backends: str) -> list[TaskConfig]:
+    """Narrow the qualification scope to an explicit backend allowlist.
+
+    Args:
+        tasks: Every task/backend bucket configured for the gate.
+        backends: Comma-separated ``backend_key`` allowlist; empty keeps all.
+
+    Returns:
+        The buckets whose backend is in the allowlist.
+
+    Raises:
+        ValueError: If a requested backend matches no configured bucket, which
+            would otherwise silently shrink the scope and weaken the verdict.
+    """
+    requested = {backend.strip() for backend in backends.split(",") if backend.strip()}
+    if not requested:
+        return tasks
+    configured = {task.backend_key for task in tasks}
+    unknown = sorted(requested - configured)
+    if unknown:
+        raise ValueError(f"Unknown backend_key(s) {unknown}; configured backends are {sorted(configured)}")
+    return [task for task in tasks if task.backend_key in requested]
+
+
 def configured_scope(
     tasks: list[TaskConfig],
     gpu_model: str,
@@ -589,6 +613,11 @@ def _parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--records", required=True, action="append", type=Path)
     parser.add_argument("--tasks_config", type=Path, default=Path(__file__).with_name("tasks.json"))
+    parser.add_argument(
+        "--backends",
+        default="",
+        help="Comma-separated backend_key allowlist to qualify (empty = every configured backend).",
+    )
     parser.add_argument("--gpu_model", default="l40s")
     parser.add_argument("--expected_target_branch")
     parser.add_argument("--expected_commit")
@@ -606,7 +635,8 @@ def main() -> int:
     """Run runner-pool stability qualification."""
     args = _parse_args()
     records = _load_records(args.records)
-    expected, noise_floors = configured_scope(load_tasks(args.tasks_config), args.gpu_model)
+    tasks = select_backends(load_tasks(args.tasks_config), args.backends)
+    expected, noise_floors = configured_scope(tasks, args.gpu_model)
     report, markdown = build_report(
         records,
         expected_buckets=expected,

tools/perf_smoke_test/test/test_runner_stability.py

@@ -312,6 +312,58 @@ def test_report_states_scope_and_decision() -> None:
     assert "runner-1" in markdown
 
 
+def test_backend_allowlist_narrows_the_qualified_scope() -> None:
+    """Only the requested backends are qualified, so unrelated buckets are not demanded."""
+    tasks = runner_stability.load_tasks(_GATE_DIR / "tasks.json")
+
+    selected = runner_stability.select_backends(tasks, "newton,physx_newton_renderer")
+
+    assert {task.backend_key for task in selected} == {"newton", "physx_newton_renderer"}
+    assert len(selected) < len(tasks)
+
+
+def test_empty_backend_allowlist_keeps_every_bucket() -> None:
+    """An empty allowlist must not silently narrow the scope."""
+    tasks = runner_stability.load_tasks(_GATE_DIR / "tasks.json")
+
+    assert runner_stability.select_backends(tasks, "") == tasks
+
+
+def test_unknown_backend_is_rejected_rather_than_silently_dropped() -> None:
+    """A typo would otherwise shrink the scope and produce a weaker verdict unnoticed."""
+    tasks = runner_stability.load_tasks(_GATE_DIR / "tasks.json")
+
+    with pytest.raises(ValueError, match="Unknown backend_key"):
+        runner_stability.select_backends(tasks, "newton,nwton")
+
+
+def test_staging_workflow_qualifies_only_newton_buckets() -> None:
+    """The sampled backends and the qualified backends must be the same set."""
+    repo_root = Path(__file__).resolve().parents[3]
+    workflow = yaml.safe_load(
+        (repo_root / ".github/workflows/perf-smoke-runner-stability.yaml").read_text(encoding="utf-8")
+    )
+
+    declared = workflow["env"]["STABILITY_BACKENDS"]
+    sampled = workflow["jobs"]["stability_sample"]["with"]["backends"]
+    assert sampled == declared, "the seeder's backend list drifted from STABILITY_BACKENDS"
+
+    report_step = next(
+        step for step in workflow["jobs"]["qualify"]["steps"] if step.get("name") == "Build qualification report"
+    )
+    assert '--backends "${STABILITY_BACKENDS}"' in report_step["run"]
+
+    # Exactly the buckets that crashed on 2026-07-28, and no PhysX-only bucket.
+    tasks = runner_stability.select_backends(runner_stability.load_tasks(_GATE_DIR / "tasks.json"), declared)
+    assert {(task.task_id, task.backend_key) for task in tasks} == {
+        ("Isaac-Cartpole-Direct", "newton"),
+        ("Isaac-Velocity-Flat-G1", "newton"),
+        ("Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", "newton_rtx_renderer"),
+        ("Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", "newton_newton_renderer"),
+        ("Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct", "physx_newton_renderer"),
+    }
+
+
 def test_staging_workflow_fans_out_complete_independent_evidence() -> None:
     """One staging merge automatically gathers and qualifies five allocations."""
     repo_root = Path(__file__).resolve().parents[3]