From 5a8111a386ec4b5bc02121a329671bd5ed88597a Mon Sep 17 00:00:00 2001
From: Paul Tancre <paultancre@p-tancre.home>
Date: Thu, 5 Nov 2026 10:39:30 +0000
Subject: [PATCH] fix(dev): honor request max_concurrency

The playground and "+ Experiment" dialog now send max_concurrency to
remote eval dev servers (braintrust#24075), but evalBodySchema dropped
it, so lowering the slider had no effect on remote evals.

Parse max_concurrency from the eval request and pass it to Eval(),
capped at the evaluator's own maxConcurrency. The request can lower
the limit but never raise it, so the playground's default of 10
cannot exceed a limit set in code.

Refs: COR-288

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../devserver-request-max-concurrency.md      |  5 ++
 js/dev/server.test.ts                         | 53 +++++++++++++++++++
 js/dev/server.ts                              | 21 ++++++++
 js/dev/types.ts                               |  1 +
 4 files changed, 80 insertions(+)
 create mode 100644 .changeset/devserver-request-max-concurrency.md

diff --git a/.changeset/devserver-request-max-concurrency.md b/.changeset/devserver-request-max-concurrency.md
new file mode 100644
index 000000000..57eb8ab99
--- /dev/null
+++ b/.changeset/devserver-request-max-concurrency.md
@@ -0,0 +1,5 @@
+---
+"braintrust": patch
+---
+
+fix(dev): honor max_concurrency from remote eval requests, capped by the evaluator's maxConcurrency
diff --git a/js/dev/server.test.ts b/js/dev/server.test.ts
index 2e1461a7e..2762cb870 100644
--- a/js/dev/server.test.ts
+++ b/js/dev/server.test.ts
@@ -19,3 +19,56 @@ describe("remote eval data", () => {
     });
   });
 });
+
+describe("remote eval max concurrency", () => {
+  test.each([
+    { evaluator: 5, request: 2, expected: 2 },
+    { evaluator: 5, request: 10, expected: 5 },
+    { evaluator: undefined, request: 3, expected: 3 },
+    { evaluator: 5, request: null, expected: 5 },
+    { evaluator: 5, request: undefined, expected: 5 },
+    { evaluator: undefined, request: undefined, expected: undefined },
+  ])(
+    "evaluator $evaluator with request $request resolves to $expected",
+    ({ evaluator, request, expected }) => {
+      expect(
+        _exportsForTestingOnly.resolveMaxConcurrency(evaluator, request),
+      ).toBe(expected);
+    },
+  );
+
+  test("parses max_concurrency from the request body", () => {
+    expect(
+      evalBodySchema.parse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: 2,
+      }).max_concurrency,
+    ).toBe(2);
+  });
+
+  test("rounds down a fractional max_concurrency", () => {
+    expect(
+      evalBodySchema.parse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: 2.5,
+      }).max_concurrency,
+    ).toBe(2);
+  });
+
+  test.each([0, -1, 0.5, "2"])(
+    "rejects invalid max_concurrency %s",
+    (maxConcurrency) => {
+      const result = evalBodySchema.safeParse({
+        name: "remote-evaluator",
+        data: { experiment_name: "source-experiment" },
+        max_concurrency: maxConcurrency,
+      });
+      expect(result.success).toBe(false);
+      expect(result.error?.issues.map((issue) => issue.path)).toEqual([
+        ["max_concurrency"],
+      ]);
+    },
+  );
+});
diff --git a/js/dev/server.ts b/js/dev/server.ts
index 1ae70a61c..c8075cf56 100644
--- a/js/dev/server.ts
+++ b/js/dev/server.ts
@@ -146,6 +146,7 @@ export function runDevServer(
         data,
         scores,
         stream,
+        max_concurrency,
       } = evalBodySchema.parse(req.body);
 
       if (!req.ctx?.state) {
@@ -229,6 +230,10 @@ export function runDevServer(
             state,
             experimentName: experiment_name ?? undefined,
             projectId: project_id ?? undefined,
+            maxConcurrency: resolveMaxConcurrency(
+              evaluator.maxConcurrency,
+              max_concurrency,
+            ),
           },
           {
             // Avoid printing the bar to the console.
@@ -340,6 +345,21 @@ async function getDataset(
   }
 }
 
+// The request can lower the evaluator's maxConcurrency but never raise it, so
+// the playground's default slider value can't exceed a limit set in code.
+function resolveMaxConcurrency(
+  evaluatorMaxConcurrency: number | undefined,
+  requestMaxConcurrency: number | null | undefined,
+): number | undefined {
+  if (requestMaxConcurrency == null) {
+    return evaluatorMaxConcurrency;
+  }
+  if (evaluatorMaxConcurrency === undefined) {
+    return requestMaxConcurrency;
+  }
+  return Math.min(evaluatorMaxConcurrency, requestMaxConcurrency);
+}
+
 function makeScorer(
   state: BraintrustState,
   name: string,
@@ -379,4 +399,5 @@ function makeScorer(
 
 export const _exportsForTestingOnly = {
   getDataset,
+  resolveMaxConcurrency,
 };
diff --git a/js/dev/types.ts b/js/dev/types.ts
index da992efea..bac98a5ac 100644
--- a/js/dev/types.ts
+++ b/js/dev/types.ts
@@ -24,6 +24,7 @@ export const evalBodySchema = z.object({
   project_id: z.string().nullish(),
   parent: invokeParentSchema.optional(),
   stream: z.boolean().optional(),
+  max_concurrency: z.number().min(1).transform(Math.floor).nullish(),
 });
 
 export type EvaluatorManifest = Record<