Merge pull request #29 from Samurai007AK/fix/26-no-web-web-are-inverted-by-the-public

fix(benchmarks): stop inverting --web/--no-web in the public runner

Author: dq-ai-dev

benchmarks/public/runner/run_subprocess.py

@@ -222,11 +222,20 @@ async def run_eval(
     # Both names are set because the toggle is read per workflow — stateful
     # reads REACT_NO_WEB, agent-team reads SWARM_NO_WEB — and a benchmark should
     # get the same policy whichever workflow runs it.
+    # --web/--no-web are open-book flags; resolve_closed_book's override is the
+    # closed-book decision, returned verbatim. Negate, or both flags do the
+    # reverse of their help text. None falls through to the benchmark's own
+    # declaration.
     from benchmarks.public.sandbox_profiles import resolve_closed_book
-    closed = resolve_closed_book(args.benchmark, getattr(args, "web", None))
+
+    web = getattr(args, "web", None)
+    closed = resolve_closed_book(
+        args.benchmark,
+        None if web is None else not web,
+    )
     os.environ["REACT_NO_WEB"] = "1" if closed else "0"
     os.environ["SWARM_NO_WEB"] = "1" if closed else "0"
-    source = "--web/--no-web" if getattr(args, "web", None) is not None else "benchmark default"
+    source = "--web/--no-web" if web is not None else "benchmark default"
     logger.info(
         "Book policy: %s (%s) — web tools %s",
         "closed-book" if closed else "open-book", source,

tests/test_benchmark_sandbox_profiles.py

@@ -92,6 +92,41 @@ def test_book_policy_override_wins_both_ways():
     assert resolve_closed_book("browsecomp", True) is True    # --no-web
 
 
+async def test_runner_does_not_invert_the_web_flags(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """``--no-web`` must reach ``resolve_closed_book`` as closed-book, and vice versa.
+
+    ``resolve_closed_book`` is polarity-correct on its own, so its unit tests
+    above stay green even when the runner hands it an un-negated *open-book*
+    flag. That is exactly how the flags shipped inverted, so the regression has
+    to be pinned on the handoff inside ``run_eval``.
+    """
+    import argparse
+
+    from benchmarks.public import sandbox_profiles
+    from benchmarks.public.runner import run_subprocess
+
+    class _Stop(Exception):
+        """Sentinel: end the run once the override has been observed."""
+
+    seen: list[bool | None] = []
+
+    def _spy(benchmark: str, override: bool | None = None) -> bool:
+        seen.append(override)
+        raise _Stop
+
+    monkeypatch.setattr(sandbox_profiles, "resolve_closed_book", _spy)
+
+    for web in (True, False, None):
+        args = argparse.Namespace(benchmark="browsecomp", pipeline=None, web=web)
+        with pytest.raises(_Stop):
+            await run_subprocess.run_eval(args, out_dir=tmp_path, seed=0)
+
+    # --web is open-book, --no-web is closed-book, no flag defers to the benchmark.
+    assert seen == [False, True, None]
+
+
 class _FakeResourceManager:
     """Minimal stand-in exposing only what the tool resolvers touch."""