Single llm() failure in llm_batch() should not error out the whole batch

Author: alexzhang13

docs/api/rlm.md

@@ -550,7 +550,7 @@ The following functions are available to model-generated code inside the REPL:
 | Function | Description |
 |:---------|:------------|
 | `llm_query(prompt, model=None)` | Single plain LM completion. Fast, no REPL or iteration. |
-| `llm_query_batched(prompts, model=None)` | Multiple plain LM completions concurrently. |
+| `llm_query_batched(prompts, model=None)` | Multiple plain LM completions concurrently. A single failed call doesn't fail the batch — that slot returns `"Error: llm() call failed - <msg>"`, the rest return normally. |
 | `rlm_query(prompt, model=None)` | Spawn a child RLM with its own REPL for deeper thinking. Falls back to `llm_query` at max depth. |
 | `rlm_query_batched(prompts, model=None)` | Spawn multiple child RLMs. Falls back to `llm_query_batched` at max depth. |
 | `answer` | A dict (`{"content": "", "ready": False}`). Set `answer["content"]` to your final answer and `answer["ready"] = True` to terminate the run. |

docs/architecture.md

@@ -227,6 +227,12 @@ all prompts as a single batched request to the handler, which processes them
 concurrently with `asyncio.gather`. `rlm_query_batched` calls `subcall_fn`
 sequentially for each prompt (each child RLM is a blocking call).
 
+**Failure handling is per-prompt.** A single failed call does not fail the
+whole batch — that slot returns the string `"Error: llm() call failed - <msg>"`
+(with the underlying error message) while the other prompts return their real
+responses. Results stay aligned with the input prompts by index, so the list is
+always the same length as `prompts`.
+
 ---
 
 ## Recursive Sub-Calls (Depth > 1)

rlm/core/comms_utils.py

@@ -255,9 +255,12 @@ def send_lm_request_batched(
         if response.chat_completions is None:
             return [LMResponse.error_response("No completions returned")] * len(prompts)
 
-        # Convert batched response to list of individual responses
+        # Convert batched response to list of individual responses. A completion
+        # carrying an error means only that prompt failed; the rest still succeed.
         return [
-            LMResponse.success_response(chat_completion)
+            LMResponse.error_response(chat_completion.error)
+            if chat_completion.error
+            else LMResponse.success_response(chat_completion)
             for chat_completion in response.chat_completions
         ]
     except Exception as e:

rlm/core/lm_handler.py

@@ -93,7 +93,9 @@ async def run_one(prompt: str):
 
         async def run_all():
             tasks = [run_one(prompt) for prompt in request.prompts]
-            return await asyncio.gather(*tasks)
+            # return_exceptions=True so one failed call doesn't abort the whole
+            # batch; failures are surfaced per-prompt as error completions below.
+            return await asyncio.gather(*tasks, return_exceptions=True)
 
         results = asyncio.run(run_all())
         end_time = time.perf_counter()
@@ -103,16 +105,31 @@ async def run_all():
         root_model = request.model or client.model_name
         usage_summary = UsageSummary(model_usage_summaries={root_model: model_usage})
 
-        chat_completions = [
-            RLMChatCompletion(
-                root_model=root_model,
-                prompt=prompt,
-                response=content,
-                usage_summary=usage_summary,
-                execution_time=total_time / len(request.prompts),  # approximate per-prompt time
-            )
-            for prompt, content in zip(request.prompts, results, strict=True)
-        ]
+        chat_completions = []
+        for prompt, content in zip(request.prompts, results, strict=True):
+            if isinstance(content, BaseException):
+                # Per-prompt failure: this slot returns an error; other prompts
+                # still succeed. The error message is carried back to the caller.
+                chat_completions.append(
+                    RLMChatCompletion(
+                        root_model=root_model,
+                        prompt=prompt,
+                        response="",
+                        usage_summary=UsageSummary(model_usage_summaries={}),
+                        execution_time=0.0,
+                        error=f"llm() call failed - {content}",
+                    )
+                )
+            else:
+                chat_completions.append(
+                    RLMChatCompletion(
+                        root_model=root_model,
+                        prompt=prompt,
+                        response=content,
+                        usage_summary=usage_summary,
+                        execution_time=total_time / len(request.prompts),  # approximate per-prompt time
+                    )
+                )
 
         return LMResponse.batched_success_response(chat_completions=chat_completions)
 

rlm/core/types.py

@@ -126,6 +126,7 @@ class RLMChatCompletion:
     metadata: dict | None = (
         None  # Full trajectory (run_metadata + iterations) when logger captures it
     )
+    error: str | None = None  # Set when this single call failed (e.g. in a batch); response is empty.
 
     def to_dict(self):
         out = {
@@ -137,6 +138,8 @@ def to_dict(self):
         }
         if self.metadata is not None:
             out["metadata"] = self.metadata
+        if self.error is not None:
+            out["error"] = self.error
         return out
 
     @classmethod
@@ -148,6 +151,7 @@ def from_dict(cls, data: dict) -> "RLMChatCompletion":
             usage_summary=UsageSummary.from_dict(data.get("usage_summary")),
             execution_time=data.get("execution_time"),
             metadata=data.get("metadata"),
+            error=data.get("error"),
         )
 
 

tests/test_lm_handler.py

@@ -30,6 +30,28 @@ def test_lm_handler_batched_request():
         assert resp.chat_completion.response == f"r{i}"
 
 
+def test_lm_handler_batched_partial_failure():
+    """One failing call returns an error for that slot; the rest still succeed."""
+
+    def response_fn(prompt):
+        if prompt == "prompt-1":
+            raise RuntimeError("boom")
+        return f"ok {prompt}"
+
+    mock = MockLM(response_fn=response_fn)
+    with LMHandler(client=mock, batch_max_concurrent=3) as handler:
+        prompts = ["prompt-0", "prompt-1", "prompt-2"]
+        result = send_lm_request_batched(handler.address, prompts)
+
+    assert len(result) == 3
+    assert result[0].success
+    assert result[0].chat_completion.response == "ok prompt-0"
+    assert not result[1].success
+    assert "boom" in result[1].error
+    assert result[2].success
+    assert result[2].chat_completion.response == "ok prompt-2"
+
+
 def test_lm_handler_batched_many_prompts_semaphore_cap():
     """Many prompts complete successfully with semaphore limiting concurrency."""
     # 50 prompts, max 4 concurrent: should still all complete