Skip to content

Commit 3519cc9

Browse files
committed
feat: enforce per-rule evidence budgets
1 parent 6f1cdef commit 3519cc9

16 files changed

Lines changed: 2657 additions & 74 deletions

‎benchmark/agent_environment.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
"""Run mini-swe shell actions as the dedicated unprivileged agent identity."""
2+
from __future__ import annotations
3+
4+
import os
5+
import subprocess
6+
7+
from benchmark.process_control import ProcessLimits, run_capped_process
8+
9+
10+
def run_as_agent(
11+
command: str,
12+
*,
13+
cwd: str,
14+
env: dict[str, str],
15+
timeout: int,
16+
uid: int,
17+
gid: int,
18+
extra_groups: tuple[int, ...] = (),
19+
max_output_chars: int = 10_000,
20+
) -> subprocess.CompletedProcess[str]:
21+
"""Execute one shell action after irreversibly dropping child privileges."""
22+
if os.name != "posix":
23+
raise RuntimeError("the rankable agent environment requires POSIX privilege separation")
24+
if os.geteuid() != 0 and (uid != os.getuid() or gid != os.getgid()):
25+
raise PermissionError("runner must be root to launch a different agent identity")
26+
27+
result = run_capped_process(
28+
command,
29+
shell=True,
30+
cwd=cwd,
31+
env=env,
32+
timeout=timeout,
33+
max_output_chars=max_output_chars,
34+
combine_stderr=True,
35+
uid=uid,
36+
gid=gid,
37+
extra_groups=extra_groups,
38+
limits=ProcessLimits(cpu_seconds=max(1, timeout), memory_bytes=2 * 1024 ** 3,
39+
file_bytes=64 * 1024 ** 2),
40+
)
41+
if result.timed_out:
42+
raise subprocess.TimeoutExpired(command, timeout, output=result.stdout)
43+
return subprocess.CompletedProcess(command, result.returncode, stdout=result.stdout)
44+
45+
46+
def make_agent_environment(session, *, uid: int, gid: int,
47+
extra_groups: tuple[int, ...] = (), timeout: int = 300):
48+
"""Construct a pinned mini-swe LocalEnvironment with a privilege-dropping execute path."""
49+
from minisweagent.environments.local import LocalEnvironment
50+
51+
class EvidenceLocalEnvironment(LocalEnvironment):
52+
def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict:
53+
command = action.get("command", "")
54+
action_cwd = cwd or self.config.cwd or str(session.workdir)
55+
if not session.admit_shell_action(command):
56+
output = {
57+
"output": "shell action budget exhausted\n",
58+
"returncode": 75,
59+
"exception_info": "",
60+
}
61+
self._check_finished(output)
62+
return output
63+
try:
64+
result = run_as_agent(
65+
command,
66+
cwd=action_cwd,
67+
env=os.environ | self.config.env,
68+
timeout=timeout or self.config.timeout,
69+
uid=uid,
70+
gid=gid,
71+
extra_groups=extra_groups,
72+
max_output_chars=session.budget.max_output_chars,
73+
)
74+
output = {"output": result.stdout, "returncode": result.returncode,
75+
"exception_info": ""}
76+
except Exception as error:
77+
raw_output = getattr(error, "output", None)
78+
raw_output = (raw_output.decode("utf-8", errors="replace")
79+
if isinstance(raw_output, bytes) else (raw_output or ""))
80+
output = {
81+
"output": raw_output,
82+
"returncode": -1,
83+
"exception_info": f"An error occurred while executing the command: {error}",
84+
"extra": {"exception_type": type(error).__name__,
85+
"exception": str(error)},
86+
}
87+
self._check_finished(output)
88+
return output
89+
90+
return EvidenceLocalEnvironment(cwd=str(session.workdir), timeout=timeout)

‎benchmark/agent_pred.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env python3
2+
"""Thin agent-facing client for the evaluation-owned pred gateway."""
3+
from __future__ import annotations
4+
5+
import json
6+
import os
7+
import sys
8+
from benchmark.agent_spool import spool_request
9+
10+
DEFAULT_REQUEST_TIMEOUT_SECONDS = 300
11+
12+
13+
def _request(payload: dict, request_id: str | None = None) -> dict:
14+
return spool_request(
15+
payload,
16+
channel_env="PRB_PRED_DIR",
17+
timeout_env="PRB_PRED_TIMEOUT",
18+
default_timeout_seconds=DEFAULT_REQUEST_TIMEOUT_SECONDS,
19+
service_name="pred gateway",
20+
request_id=request_id,
21+
)
22+
23+
24+
def _format_status(budget: dict) -> str:
25+
pred = budget["pred_calls"]
26+
solve = budget["solve_calls"]
27+
return (f"pred calls: {pred['used']}/{pred['limit']} used, {pred['remaining']} remaining; "
28+
f"solve calls: {solve['used']}/{solve['limit']} used, "
29+
f"{solve['remaining']} remaining")
30+
31+
32+
def main() -> None:
33+
args = sys.argv[1:]
34+
try:
35+
if args == ["--budget-status"]:
36+
response = _request({"op": "status", "cwd": os.getcwd()})
37+
print(_format_status(response["budget"]))
38+
return
39+
response = _request({"op": "pred", "args": args, "cwd": os.getcwd()})
40+
except (OSError, ValueError, json.JSONDecodeError, RuntimeError, TimeoutError) as error:
41+
print(f"pred unavailable: {error}", file=sys.stderr)
42+
raise SystemExit(2)
43+
44+
if response.get("stdout"):
45+
print(response["stdout"], end="" if response["stdout"].endswith("\n") else "\n")
46+
if response.get("stderr"):
47+
print(response["stderr"], end="" if response["stderr"].endswith("\n") else "\n",
48+
file=sys.stderr)
49+
raise SystemExit(int(response.get("returncode", 2)))
50+
51+
52+
if __name__ == "__main__":
53+
main()

‎benchmark/agent_spool.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"""Shared thin-client transport for evaluation-owned agent commands."""
2+
from __future__ import annotations
3+
4+
import json
5+
import os
6+
import time
7+
import uuid
8+
from pathlib import Path
9+
10+
POLL_INTERVAL_SECONDS = 0.02
11+
12+
13+
def spool_request(
14+
payload: dict,
15+
*,
16+
channel_env: str,
17+
timeout_env: str,
18+
default_timeout_seconds: int,
19+
service_name: str,
20+
request_id: str | None = None,
21+
) -> dict:
22+
"""Publish one idempotent request and await its atomic response."""
23+
channel = os.environ.get(channel_env)
24+
if not channel:
25+
raise RuntimeError(
26+
f"{service_name} is only available inside an active benchmark evaluation")
27+
request_id = request_id or os.environ.get("PRB_REQUEST_ID") or uuid.uuid4().hex
28+
if len(request_id) != 32 or any(char not in "0123456789abcdef" for char in request_id):
29+
raise ValueError("request id must be 32 lowercase hexadecimal characters")
30+
31+
channel_dir = Path(channel)
32+
inbox = channel_dir / "inbox"
33+
outbox = channel_dir / "outbox"
34+
request_path = inbox / f"{request_id}.json"
35+
response_path = outbox / f"{request_id}.json"
36+
envelope = {"request_id": request_id, **payload}
37+
38+
timeout = int(os.environ.get(timeout_env, default_timeout_seconds))
39+
attempts = int(os.environ.get("PRB_CLIENT_RETRIES", "1")) + 1
40+
for attempt in range(attempts):
41+
temporary = inbox / f".{request_id}.{os.getpid()}.{attempt}.tmp"
42+
temporary.write_text(json.dumps(envelope), encoding="utf-8")
43+
os.replace(temporary, request_path)
44+
deadline = time.monotonic() + timeout
45+
while time.monotonic() < deadline:
46+
try:
47+
raw = response_path.read_text(encoding="utf-8")
48+
except FileNotFoundError:
49+
time.sleep(POLL_INTERVAL_SECONDS)
50+
continue
51+
response = json.loads(raw)
52+
try:
53+
response_path.unlink()
54+
except FileNotFoundError:
55+
pass
56+
return response
57+
raise TimeoutError(f"{service_name} did not respond within {timeout}s")

‎benchmark/agent_submit.py‎

Lines changed: 20 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,13 @@
99

1010
import argparse
1111
import json
12-
import os
1312
import sys
14-
import time
15-
import uuid
1613
from pathlib import Path
1714

15+
from benchmark.agent_spool import spool_request
16+
1817
MAX_CERTIFICATE_BYTES = 1024 * 1024
1918
DEFAULT_REQUEST_TIMEOUT_SECONDS = 300
20-
POLL_INTERVAL_SECONDS = 0.02
2119

2220

2321
def _read_certificate(path: str) -> str:
@@ -32,67 +30,46 @@ def _read_certificate(path: str) -> str:
3230
return raw.decode("utf-8")
3331

3432

35-
def _request(payload: dict) -> dict:
36-
channel = os.environ.get("PRB_SUBMIT_DIR")
37-
if not channel:
38-
raise RuntimeError("submit is only available inside an active benchmark evaluation")
39-
channel_dir = Path(channel)
40-
request_id = uuid.uuid4().hex
41-
inbox = channel_dir / "inbox"
42-
outbox = channel_dir / "outbox"
43-
request_path = inbox / f"{request_id}.json"
44-
temp_path = inbox / f".{request_id}.{os.getpid()}.tmp"
45-
response_path = outbox / f"{request_id}.json"
46-
47-
envelope = {"request_id": request_id, **payload}
48-
temp_path.write_text(json.dumps(envelope), encoding="utf-8")
49-
os.replace(temp_path, request_path)
50-
51-
timeout = int(os.environ.get("PRB_SUBMIT_TIMEOUT", DEFAULT_REQUEST_TIMEOUT_SECONDS))
52-
deadline = time.monotonic() + timeout
53-
while time.monotonic() < deadline:
54-
try:
55-
raw = response_path.read_text(encoding="utf-8")
56-
except FileNotFoundError:
57-
time.sleep(POLL_INTERVAL_SECONDS)
58-
continue
59-
response = json.loads(raw)
60-
try:
61-
response_path.unlink()
62-
except FileNotFoundError:
63-
pass
64-
return response
65-
raise TimeoutError(f"submit service did not respond within {timeout}s")
33+
def _request(payload: dict, request_id: str | None = None) -> dict:
34+
return spool_request(
35+
payload,
36+
channel_env="PRB_SUBMIT_DIR",
37+
timeout_env="PRB_SUBMIT_TIMEOUT",
38+
default_timeout_seconds=DEFAULT_REQUEST_TIMEOUT_SECONDS,
39+
service_name="submit service",
40+
request_id=request_id,
41+
)
6642

6743

6844
def main() -> None:
6945
parser = argparse.ArgumentParser(
7046
prog="submit",
7147
description="Submit one rule counterexample to the benchmark verifier. Every "
72-
"submission, accepted or rejected, consumes one run-wide attempt.",
48+
"submission, accepted or rejected, consumes one rule-scoped attempt.",
7349
)
7450
parser.add_argument("certificate", nargs="?", help="Certificate JSON file, or - for stdin")
7551
parser.add_argument("--status", action="store_true", help="Show remaining attempts (free)")
52+
parser.add_argument("--request-id", help=argparse.SUPPRESS)
7653
args = parser.parse_args()
7754

7855
try:
7956
if args.status:
8057
if args.certificate:
8158
parser.error("--status does not take a certificate")
82-
response = _request({"op": "status"})
59+
response = _request({"op": "status"}, args.request_id)
8360
print(f"submit budget: {response['used']}/{response['limit']} used, "
8461
f"{response['remaining']} remaining")
8562
return
8663
if not args.certificate:
8764
parser.error("a certificate JSON file is required (use - for stdin)")
8865
try:
8966
text = _read_certificate(args.certificate)
90-
response = _request({"op": "submit", "certificate_text": text})
67+
response = _request({"op": "submit", "certificate_text": text}, args.request_id)
9168
except (OSError, UnicodeError, ValueError) as e:
9269
# File/encoding/size failures are still submission attempts; let the service
9370
# account for them instead of failing locally without consuming budget.
94-
response = _request({"op": "submit", "client_error": str(e)})
95-
except (OSError, ValueError, json.JSONDecodeError, RuntimeError) as e:
71+
response = _request({"op": "submit", "client_error": str(e)}, args.request_id)
72+
except (OSError, ValueError, json.JSONDecodeError, RuntimeError, TimeoutError) as e:
9673
print(f"submit unavailable: {e}", file=sys.stderr)
9774
raise SystemExit(2)
9875

@@ -102,9 +79,10 @@ def main() -> None:
10279
return
10380

10481
reason = response.get("reason") or response.get("error") or "rejected"
105-
label = "BUDGET_EXHAUSTED" if response.get("exhausted") else "REJECTED"
82+
label = ("INFRASTRUCTURE_ERROR" if response.get("infrastructure_error") else
83+
"BUDGET_EXHAUSTED" if response.get("exhausted") else "REJECTED")
10684
print(f"{label} attempt {used}/{limit} ({remaining} remaining): {reason}", file=sys.stderr)
107-
raise SystemExit(2 if response.get("exhausted") else 1)
85+
raise SystemExit(2 if response.get("exhausted") or response.get("infrastructure_error") else 1)
10886

10987

11088
if __name__ == "__main__":

‎benchmark/backend_score.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import hashlib
2020
import json
2121
import re
22-
import shutil
2322
import sys
2423
from pathlib import Path
2524

@@ -123,10 +122,12 @@ def _assert_pred_version() -> None:
123122
"""The backend is the authoritative verifier, so its pred must be the pinned version.
124123
Skip when no pred is on PATH (pred-free unit tests never verify real certificates) or
125124
when EXPECTED_PRED_VERSION is set empty; otherwise hard-fail on a mismatch."""
126-
if not shutil.which("pred"):
125+
from benchmark.env_setup import find_pred_binary, verify_pred_version
126+
try:
127+
pred_binary = find_pred_binary()
128+
except RuntimeError:
127129
return
128-
from benchmark.env_setup import verify_pred_version
129-
verify_pred_version("pred") # raises ValueError on mismatch
130+
verify_pred_version(pred_binary) # raises ValueError on mismatch
130131

131132

132133
# ── status helpers ────────────────────────────────────────────────────────────

0 commit comments

Comments
 (0)