feat: add private research controls on top of URL denylist

docs/reference/cli.md

@@ -262,6 +262,7 @@ bench eval run --tasks-dir ./tasks --matrix matrix.yaml --trials 3
 | `--skill-mode` | `no-skill` | Skill mode: `no-skill`, `with-skill`, or `self-gen` |
 | `--skill-creator-dir` | — | Path to a `skill-creator` directory (or a skills root containing it); used when `--skill-mode self-gen` |
 | `--self-gen-no-internet` | `false` | Disable web tools for the self-generated skill run |
+| `--research-policy` | — | Apply a private per-task filtered-research policy (Docker only); see [Filtered web research](../research-policy.md) |
 | `--agent-env` | — | Agent environment variable as `KEY=VALUE`; repeatable |
 | `--include` | — | Only run these task names; repeatable (e.g. `--include jax-computing-basics --include data-to-d3`) |
 | `--exclude` | — | Skip these task names; repeatable (e.g. `--exclude quantum-numerical-simulation`) |

docs/research-policy.md

@@ -0,0 +1,80 @@
+# Filtered web research
+
+Some research benchmarks need ordinary Internet access while keeping a small
+set of papers or answer-bearing pages unavailable. Use a private, run-scoped
+research policy:
+
+```bash
+bench eval run \
+  --tasks-dir tasks/my-research-task \
+  --agent codex-acp \
+  --sandbox docker \
+  --research-policy /secure/frontierphysics-policy.yaml
+```
+
+The policy is deliberately not part of the task package. Do not commit it next
+to `task.md`, put its values in a prompt, or pass it through `--agent-env`.
+
+## Policy format
+
+```yaml
+version: 1
+tasks:
+  my-research-task:
+    blocked_urls:
+      - https://example.org/papers/answer.html
+    blocked_url_prefixes:
+      - https://example.org/supplements/answer
+    blocked_hosts:
+      - private-corpus.example.org
+    blocked_terms:
+      - Exact Paper Title
+      - 10.1234/example.doi
+    blocked_content_sha256:
+      - 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+```
+
+Every selected task needs its own entry. A missing task or an entry that blocks
+nothing stops the run. `blocked_urls` ignores query strings, fragments, and an
+HTTP/HTTPS scheme change. `blocked_url_prefixes` also covers descendant paths.
+`blocked_hosts` includes subdomains. Terms are case-insensitive and filter both
+search-result titles and fetched text; content hashes cover exact binary or text
+bodies.
+
+The optional top-level `search_endpoint` selects an HTTP(S) search HTML
+endpoint. It defaults to DuckDuckGo's Lite HTML endpoint.
+
+## Enforcement model
+
+On Docker, BenchFlow disables each harness's native web tools and provides the
+same `benchflow-research` MCP server to ACP and native-MCP-config agents. It
+offers `web_search`, `web_fetch`, and `web_download`. The server talks only to a
+root-owned loopback gateway, which checks the destination, every redirect, and
+the returned content.
+
+The model provider uses a separate loopback proxy. Before the agent process is
+launched, an owner-based IPv4/IPv6 firewall blocks every other connection from
+the sandbox user. This prevents `curl`, sockets, or an unregistered harness tool
+from bypassing the gateway. Private, loopback, link-local, and other non-global
+fetch destinations are rejected to prevent SSRF.
+
+Claude subscription authentication cannot be translated through LiteLLM because
+there is no operator-owned API key. In that mode, the research gateway also
+provides a fixed-destination loopback relay for the native Anthropic protocol.
+The relay forwards only to `api.anthropic.com`; it is not a general HTTP proxy,
+and rejects provider-side web-search/web-fetch tools plus remote MCP requests,
+so the sandbox user remains unable to connect directly to research sites.
+
+Policy-enabled runs currently require Docker, Python 3 in the task image, and a
+non-root `sandbox_user`. Unsupported sandboxes and already-started external
+sandboxes fail closed.
+
+`config.json` records only the resolved policy SHA-256, rule counts, and whether
+the gateway plus firewall became active. The private path and all rule values
+are omitted from durable worker payloads and rollout artifacts.
+
+This mechanism can guarantee that the sandbox cannot directly retrieve the
+listed resources. It cannot guarantee that a model has never seen a paper in
+pretraining or that an unlisted mirror/citation cannot reveal its existence.
+Use terms, content hashes, and URLs for known mirrors when discovery leakage is
+part of the benchmark threat model.

src/benchflow/cli/eval_artifacts.py

@@ -40,6 +40,7 @@ def _redacted_eval_config(eval_config: EvaluationConfig) -> dict:
         "source_provenance": eval_config.source_provenance,
         "dataset_name": eval_config.dataset_name,
         "dataset_version": eval_config.dataset_version,
+        "research_policy": eval_config.research_policy_path is not None,
     }
 
 

src/benchflow/cli/main.py

@@ -451,6 +451,16 @@ def eval_run(
             help="Disable web tools for the self-generated run",
         ),
     ] = False,
+    research_policy: Annotated[
+        Path | None,
+        typer.Option(
+            "--research-policy",
+            help=(
+                "Private YAML policy for filtered web research. Docker only; "
+                "the policy contents are not copied into run artifacts."
+            ),
+        ),
+    ] = None,
     loop_strategy: Annotated[
         str | None,
         typer.Option(
@@ -658,6 +668,7 @@ def eval_run(
         skill_mode=skill_mode,
         skill_creator_dir=skill_creator_dir,
         self_gen_no_internet=self_gen_no_internet,
+        research_policy=research_policy,
         loop_strategy=loop_strategy,
         agent_env=_parse_agent_env(agent_env),
         include=include,
@@ -966,6 +977,8 @@ def _run_config_file_eval(plan: "EvalPlan") -> None:
         # run-config file was a no-op.
         if plan.eval_config_override is not None:
             j._config.config_override = plan.eval_config_override
+        if req.research_policy is not None:
+            j._config.research_policy_path = str(req.research_policy.resolve())
     except subprocess.CalledProcessError as e:
         # A source.repo clone/fetch failure (git exits non-zero) otherwise escapes
         # as a raw traceback — it is not a config-parse error, so give it its own

src/benchflow/contracts/planes.py

@@ -108,6 +108,9 @@ async def install_agent(
     async def write_credential_files(self, *args: Any, **kwargs: Any) -> None: ...
     async def upload_subscription_auth(self, *args: Any, **kwargs: Any) -> None: ...
     async def apply_web_tool_policy(self, *args: Any, **kwargs: Any) -> None: ...
+    async def enforce_agent_egress_firewall(
+        self, env: Any, sandbox_user: str | None, agent_env: dict[str, str]
+    ) -> None: ...
     async def link_skill_paths(self, *args: Any, **kwargs: Any) -> None: ...
     async def ensure_litellm_runtime(self, *args: Any, **kwargs: Any) -> Any: ...
     async def stop_provider_runtime(self, runtime: Any) -> None: ...

src/benchflow/eval_plan.py

@@ -106,6 +106,7 @@ class EvalCreateRequest:
     skill_mode: str = SKILL_MODE_NO_SKILL
     skill_creator_dir: Path | None = None
     self_gen_no_internet: bool = False
+    research_policy: Path | None = None
     loop_strategy: str | None = None
     agent_env: dict[str, str] = field(default_factory=dict)
     include: list[str] | None = None
@@ -204,6 +205,9 @@ def make_eval_config(
                 str(req.skill_creator_dir) if req.skill_creator_dir else None
             ),
             self_gen_no_internet=req.self_gen_no_internet,
+            research_policy_path=(
+                str(req.research_policy.resolve()) if req.research_policy else None
+            ),
             source_provenance=source_provenance,
             dataset_name=dataset_name,
             dataset_version=dataset_version,
@@ -258,6 +262,13 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan:
         )
     if request.registry and not request.dataset:
         raise EvalPlanError("--registry requires --dataset")
+    if request.research_policy is not None:
+        if request.source_env:
+            raise EvalPlanError("--research-policy is not supported with --source-env")
+        if not request.research_policy.is_file():
+            raise EvalPlanError(
+                f"Research policy file does not exist: {request.research_policy}"
+            )
     if request.ignore_bench_version and not request.dataset:
         raise EvalPlanError("--ignore-bench-version requires --dataset")
     if request.matrix is not None and not request.tasks_dir:
@@ -364,8 +375,12 @@ def build_eval_plan(request: EvalCreateRequest) -> EvalPlan:
                     "Missing optional dependency for 'modal' sandbox. "
                     f"Install it with `uv sync --extra {provider_extra('modal')}`."
                 ) from exc
+        if request.research_policy is not None and eval_environment != "docker":
+            raise EvalPlanError("--research-policy currently requires --sandbox docker")
     eval_prompts = cast("list[str | None] | None", request.prompt)
     sandbox_user = normalize_sandbox_user(request.sandbox_user)
+    if request.research_policy is not None and sandbox_user is None:
+        raise EvalPlanError("--research-policy requires a non-root --sandbox-user")
     eval_concurrency = request.concurrency if request.concurrency is not None else 4
     if eval_concurrency < 1:
         # A non-positive concurrency builds asyncio.Semaphore(0), which can never

src/benchflow/eval_sharding.py

@@ -138,6 +138,7 @@ def _config_payload(
         "skill_mode": config.skill_mode,
         "skill_creator_dir": config.skill_creator_dir,
         "self_gen_no_internet": config.self_gen_no_internet,
+        "research_policy_path": config.research_policy_path,
         "job_mode": config.job_mode,
         "source_provenance": config.source_provenance,
         # Serialize the already-resolved manifest OBJECT (the S axis), not a
@@ -163,6 +164,10 @@ def _config_payload(
 
 def _redacted_config_payload(config_payload: dict[str, Any]) -> dict[str, Any]:
     artifact_payload = dict(config_payload)
+    # Workers receive this through a mode-0600 temporary payload, but the
+    # durable worker_payload.json must not disclose the private policy path.
+    if artifact_payload.get("research_policy_path") is not None:
+        artifact_payload["research_policy_path"] = "<private>"
     agent_env = artifact_payload.get("agent_env")
     if isinstance(agent_env, dict):
         artifact_payload["agent_env"] = {

src/benchflow/eval_worker.py

@@ -72,6 +72,7 @@ def _evaluation_config(raw: dict[str, Any]) -> EvaluationConfig:
         skill_mode=raw.get("skill_mode") or SKILL_MODE_NO_SKILL,
         skill_creator_dir=raw.get("skill_creator_dir"),
         self_gen_no_internet=bool(raw.get("self_gen_no_internet", False)),
+        research_policy_path=raw.get("research_policy_path"),
         job_mode=raw.get("job_mode") or "parallel-independent",
         source_provenance=raw.get("source_provenance"),
         usage_tracking=UsageTrackingConfig.from_mapping(raw),

src/benchflow/evaluation.py

@@ -503,6 +503,9 @@ class EvaluationConfig:
     skill_mode: str = SKILL_MODE_NO_SKILL
     skill_creator_dir: str | None = None
     self_gen_no_internet: bool = False
+    # Private operator-side policy file. Its contents and path are never copied
+    # into public rollout/worker artifacts.
+    research_policy_path: str | None = None
     job_mode: str = DEFAULT_JOB_MODE
     source_provenance: dict[str, Any] | None = None
     # Registry dataset identity (`bench eval run -d name@version`). When
@@ -914,6 +917,7 @@ def _from_native_yaml(cls, raw: dict, **kwargs) -> Evaluation:
                 else None
             ),
             self_gen_no_internet=bool(raw.get("self_gen_no_internet", False)),
+            research_policy_path=raw.get("research_policy"),
             job_mode=raw.get("job_mode", DEFAULT_JOB_MODE),
             source_provenance=source_provenance,
             usage_tracking=UsageTrackingConfig.from_mapping(raw),
@@ -1304,6 +1308,7 @@ async def _run_single_task(
             skill_mode=skill_mode,
             skill_creator_dir=cfg.skill_creator_dir,
             self_gen_no_internet=cfg.self_gen_no_internet,
+            research_policy_path=cfg.research_policy_path,
             export_generated_skills_to=export_to,
             source_provenance=task_source_provenance(cfg.source_provenance, task_dir),
             dataset=dataset,
@@ -1364,6 +1369,7 @@ async def _run_single_task_legacy(
             skill_mode=cfg.skill_mode,
             skill_creator_dir=cfg.skill_creator_dir,
             self_gen_no_internet=cfg.self_gen_no_internet,
+            research_policy_path=cfg.research_policy_path,
             source_provenance=task_source_provenance(cfg.source_provenance, task_dir),
             usage_tracking=cfg.usage_tracking,
         )

src/benchflow/providers/litellm_config.py

@@ -404,14 +404,21 @@ def resolve_litellm_route(model: str, env: dict[str, str]) -> LiteLLMRoute:
         required = ("OPENAI_API_KEY",)
 
     params: dict[str, str | int | float | bool | list[str]] = {"model": upstream}
-    if upstream.lower().startswith("gemini/"):
-        explicit_api_base = (env.get("BENCHFLOW_PROVIDER_BASE_URL") or "").strip()
-        explicit_api_key = (env.get("BENCHFLOW_PROVIDER_API_KEY") or "").strip()
-        if explicit_api_base:
-            params["api_base"] = explicit_api_base
-            if explicit_api_key:
-                params["api_key"] = _env_ref("BENCHFLOW_PROVIDER_API_KEY")
-                required = ("BENCHFLOW_PROVIDER_API_KEY",)
+    explicit_api_base = (env.get("BENCHFLOW_PROVIDER_BASE_URL") or "").strip()
+    explicit_api_key = (env.get("BENCHFLOW_PROVIDER_API_KEY") or "").strip()
+    if explicit_api_base:
+        # Explicit generic endpoints also apply to unregistered/bare model IDs.
+        # Without this, a future or private OpenAI-compatible model silently
+        # falls back to the canonical provider even though resolve_agent_env()
+        # has accepted the operator-supplied route.
+        params["api_base"] = explicit_api_base
+        if explicit_api_key:
+            params["api_key"] = _env_ref("BENCHFLOW_PROVIDER_API_KEY")
+            required = ("BENCHFLOW_PROVIDER_API_KEY",)
+    if upstream.lower().startswith("openai/"):
+        effort = _provider_reasoning_effort(env)
+        if effort:
+            params["reasoning_effort"] = effort
 
     key = required[0] if required else None
     if key and "api_key" not in params:

src/benchflow/providers/litellm_logging.py

@@ -160,6 +160,46 @@ def _gate_opencode_skill_catalog(data: dict[str, Any]) -> None:
     _skill_catalog_gate_passed = True
 
 
+def _is_server_web_tool(tool: Any) -> bool:
+    # Recognize provider-executed web tools without matching client functions.
+    if not isinstance(tool, dict):
+        return False
+    tool_type = str(tool.get("type") or "").lower().replace("-", "_")
+    if tool_type.startswith(("web_search", "web_fetch", "url_context")):
+        return True
+    normalized_keys = {str(key).lower().replace("-", "_") for key in tool}
+    return bool(
+        normalized_keys
+        & {
+            "google_search",
+            "google_search_retrieval",
+            "web_search",
+            "web_search_preview",
+            "url_context",
+        }
+    )
+
+
+def _remove_server_web_fields(data: dict[str, Any]) -> dict[str, Any]:
+    # Strip confused-deputy web controls from requests to the model proxy.
+    forbidden = {
+        "google_search",
+        "google_search_retrieval",
+        "web_search",
+        "web_search_options",
+        "web_search_preview",
+        "url_context",
+    }
+    cleaned = dict(data)
+    for key in list(cleaned):
+        if str(key).lower().replace("-", "_") in forbidden:
+            cleaned.pop(key, None)
+    tools = cleaned.get("tools")
+    if isinstance(tools, list):
+        cleaned["tools"] = [tool for tool in tools if not _is_server_web_tool(tool)]
+    return cleaned
+
+
 def _jsonable(value: Any) -> Any:
     if value is None or isinstance(value, (str, int, float, bool)):
         return value
@@ -315,6 +355,19 @@ async def async_pre_call_hook(
                     cleaned = dict(data)
                 cleaned["tools"] = kept
 
+        # In no-web / filtered-research runs, the agent can reach only this
+        # loopback model proxy. Do not let that proxy become a confused deputy:
+        # strip provider-executed search/URL-context tools even when a curious
+        # agent crafts the request directly instead of using its harness. Client
+        # function tools (shell/file/MCP calls) remain available.
+        if os.environ.get("BENCHFLOW_DISALLOW_WEB_TOOLS") == "1":
+            web_cleaned = _remove_server_web_fields(cleaned)
+            extra_body = web_cleaned.get("extra_body")
+            if isinstance(extra_body, dict):
+                web_cleaned["extra_body"] = _remove_server_web_fields(extra_body)
+            if web_cleaned != cleaned:
+                cleaned = web_cleaned
+
         # Forward ``reasoning_effort`` VERBATIM on deepseek routes. LiteLLM's
         # deepseek transform consumes the top-level field (it maps it into its
         # own thinking handling and drops the raw param — even with drop_params

src/benchflow/providers/litellm_runtime.py

@@ -40,6 +40,7 @@
     LITELLM_MASTER_KEY_ENV,
     LITELLM_MODEL_ALIAS_ENV,
     LITELLM_MODEL_VIA_ENV,
+    PROVIDER_REASONING_EFFORT_ENV,
     LiteLLMRoute,
     litellm_proxy_config,
     resolve_litellm_route,
@@ -1057,6 +1058,11 @@ async def _ensure_sandbox_litellm(
     )
     command = f"""
 set -eu
+# cryptography 48's aarch64 wheel can execute an unsupported OpenSSL feature
+# probe under Docker Desktop's ARM VM and terminate with SIGILL.  LiteLLM
+# 1.91 pins cryptography to the 48.x line, so force the portable OpenSSL path
+# for the bootstrap import as well as for the long-running proxy below.
+export OPENSSL_armcap=0
 export PATH="$HOME/.local/bin:$PATH"
 UV="$(command -v uv || true)"
 if [ -z "$UV" ]; then
@@ -1214,6 +1220,7 @@ async def _start_sandbox_litellm(
         env.update(
             {
                 "PYTHONPATH": f"{runtime_dir}:{env.get('PYTHONPATH', '')}",
+                "OPENSSL_armcap": "0",
                 "LITELLM_MASTER_KEY": master_key,
                 "BENCHFLOW_LITELLM_LOG_PATH": paths["log"],
                 **_PROXY_DOCS_DISABLE_ENV,
@@ -1625,6 +1632,7 @@ async def ensure_litellm_runtime(
     model: str | None,
     runtime: Any | None,
     environment: str,
+    reasoning_effort: str | None = None,
     session_id: str = "",
     usage_tracking: UsageTrackingConfig | dict[str, Any] | str | None = None,
     sandbox: Any | None = None,
@@ -1680,7 +1688,11 @@ async def ensure_litellm_runtime(
         raise RuntimeError("sandbox-local LiteLLM requires a sandbox handle")
 
     try:
-        route = resolve_litellm_route(model, agent_env)
+        route_env = agent_env
+        if agent == "codex-acp" and reasoning_effort:
+            route_env = dict(agent_env)
+            route_env[PROVIDER_REASONING_EFFORT_ENV] = reasoning_effort
+        route = resolve_litellm_route(model, route_env)
     except ValueError as exc:
         await _raise_litellm_unavailable(
             runtime=runtime,