From d96b8a6634f3058583f2a9e58843dd3638325e93 Mon Sep 17 00:00:00 2001 From: Paul Tancre Date: Thu, 5 Nov 2026 10:39:30 +0000 Subject: [PATCH] fix(devserver): honor request max_concurrency The playground and "+ Experiment" dialog now send max_concurrency to remote eval dev servers (braintrust#24075), but run_eval ignored it, so lowering the slider had no effect on remote evals. Parse max_concurrency from the eval request and pass it to EvalAsync, capped at the evaluator's own max_concurrency. The request can lower the limit but never raise it, so the playground's default of 10 cannot exceed a limit set in code. Refs: COR-288 Co-Authored-By: Claude Opus 5.5 --- py/src/braintrust/devserver/schemas.py | 7 ++ py/src/braintrust/devserver/server.py | 10 +++ .../devserver/test_server_integration.py | 90 +++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/py/src/braintrust/devserver/schemas.py b/py/src/braintrust/devserver/schemas.py index a359a93dd..884cb99e7 100644 --- a/py/src/braintrust/devserver/schemas.py +++ b/py/src/braintrust/devserver/schemas.py @@ -42,6 +42,7 @@ class ParsedEvalBody(TypedDict, total=False): project_id: str parent: str | ParsedParent stream: bool + max_concurrency: int def validate_typed_dict(data: Any, typed_dict_class: type, path: str = "") -> dict[str, Any]: @@ -260,4 +261,10 @@ def parse_eval_body(request_data: str | bytes | dict) -> ParsedEvalBody: raise ValidationError("stream must be a boolean") parsed["stream"] = data["stream"] + if data.get("max_concurrency") is not None: + max_concurrency = data["max_concurrency"] + if not isinstance(max_concurrency, (int, float)) or isinstance(max_concurrency, bool) or max_concurrency < 1: + raise ValidationError("max_concurrency must be a number greater than or equal to 1") + parsed["max_concurrency"] = int(max_concurrency) + return parsed diff --git a/py/src/braintrust/devserver/server.py b/py/src/braintrust/devserver/server.py index af291444e..40dd6d2d2 100644 --- a/py/src/braintrust/devserver/server.py +++ b/py/src/braintrust/devserver/server.py @@ -234,6 +234,16 @@ def stream_fn(event: SSEProgressEvent): # registered against. project_id = eval_data.get("project_id") or evaluator.project_id + # The request can lower the evaluator's max_concurrency but never raise it, + # so the playground's default slider value can't exceed a limit set in code. + request_max_concurrency = eval_data.get("max_concurrency") + if request_max_concurrency is not None: + eval_kwargs["max_concurrency"] = ( + request_max_concurrency + if evaluator.max_concurrency is None + else min(request_max_concurrency, evaluator.max_concurrency) + ) + try: eval_task = asyncio.create_task( EvalAsync( diff --git a/py/src/braintrust/devserver/test_server_integration.py b/py/src/braintrust/devserver/test_server_integration.py index 7712bb820..39bcb392e 100644 --- a/py/src/braintrust/devserver/test_server_integration.py +++ b/py/src/braintrust/devserver/test_server_integration.py @@ -480,3 +480,93 @@ async def fake_eval_async(*, project_id, **_kwargs): assert response.status_code == 200 assert captured["project_id"] == "request-explicit-project-id" + + +_OMIT = object() + + +@pytest.mark.parametrize( + "evaluator_max_concurrency,request_max_concurrency,expected", + [ + pytest.param(5, 2, 2, id="request-lowers-evaluator"), + pytest.param(5, 10, 5, id="request-cannot-raise-evaluator"), + pytest.param(None, 3, 3, id="request-applies-when-evaluator-unset"), + pytest.param(5, None, 5, id="null-request-keeps-evaluator"), + pytest.param(5, _OMIT, 5, id="omitted-request-keeps-evaluator"), + pytest.param(None, _OMIT, None, id="both-unset"), + ], +) +def test_eval_request_max_concurrency_caps_evaluator( + api_key, org_name, monkeypatch, evaluator_max_concurrency, request_max_concurrency, expected +): + """The playground's max concurrency can lower, but never raise, the evaluator's own limit.""" + from braintrust import Evaluator + from braintrust.devserver import server as devserver_module + from braintrust.devserver.server import create_app + from braintrust.logger import BraintrustState + from starlette.testclient import TestClient + + evaluator = Evaluator( + project_name="test-max-concurrency", + eval_name="max-concurrency-eval", + data=lambda: [{"input": "ping", "expected": "pong"}], + task=lambda input, _hooks: "pong", + scores=[], + experiment_name=None, + metadata=None, + max_concurrency=evaluator_max_concurrency, + ) + + captured: dict[str, Any] = {} + + async def fake_cached_login(**_kwargs): + return BraintrustState() + + class FakeSummary: + def as_dict(self): + return {"experiment_name": evaluator.eval_name, "project_name": "", "scores": {}} + + class FakeResult: + summary = FakeSummary() + + async def fake_eval_async(*, max_concurrency, **_kwargs): + captured["max_concurrency"] = max_concurrency + return FakeResult() + + monkeypatch.setattr(devserver_module, "cached_login", fake_cached_login) + monkeypatch.setattr(devserver_module, "EvalAsync", fake_eval_async) + + eval_request: dict[str, Any] = { + "name": "max-concurrency-eval", + "stream": False, + "data": [{"input": "ping", "expected": "pong"}], + } + if request_max_concurrency is not _OMIT: + eval_request["max_concurrency"] = request_max_concurrency + + response = TestClient(create_app([evaluator])).post( + "/eval", + headers={ + "x-bt-auth-token": api_key, + "x-bt-org-name": org_name, + "Content-Type": "application/json", + }, + json=eval_request, + ) + + assert response.status_code == 200 + assert captured["max_concurrency"] == expected + + +def test_parse_eval_body_rounds_down_fractional_max_concurrency(): + from braintrust.devserver.schemas import parse_eval_body + + assert parse_eval_body({"name": "eval", "max_concurrency": 2.5})["max_concurrency"] == 2 + + +@pytest.mark.parametrize("max_concurrency", [0, -1, 0.5, "2", True]) +def test_parse_eval_body_rejects_invalid_max_concurrency(max_concurrency): + from braintrust.devserver.schemas import ValidationError, parse_eval_body + + with pytest.raises(ValidationError, match="max_concurrency must be a number greater than or equal to 1"): + parse_eval_body({"name": "eval", "max_concurrency": max_concurrency})