Skip to content

Commit 795f33b

Browse files
authored
fix(eval): await asynchronous data sources (#1076)
`EvalData` accepts awaitable iterators and async data factories, but both `Eval` and `EvalAsync` currently pass the resulting coroutine into a synchronous `for` loop. Evaluation fails with `TypeError: 'coroutine' object is not iterable` before processing a sample. Await the data after resolving its factory and before adapting the iterator. This matches the existing `workflow_eval.py` handling and preserves synchronous iterators and async generators. Validation on Python 3.13.18 with locked dependencies: - The new public-API regression has 4 failures and 4 passing synchronous controls on base `e26deb0`; all 8 cases pass with the fix. - `pytest src/braintrust/test_framework.py src/braintrust/test_framework2.py --vcr-record=none -q`: 180 passed, 5 skipped. - Ruff 0.15.21 lint and format checks pass for both changed files. - Full `pre-commit run --all-files` passes (pre-commit 4.6.2, including Ruff and Codespell). - Targeted Pylint (`--errors-only`) passes for both changed files. - Full `nox -R -s test_core -- --vcr-record=none -q`: 1,127 passed, 114 skipped, 12 xfailed, 23 subtests passed. Git signing was disabled only in this test process for deterministic generated Git fixtures.
1 parent 2651d7a commit 795f33b

2 files changed

Lines changed: 43 additions & 0 deletions

File tree

‎py/src/braintrust/framework.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2014,6 +2014,9 @@ async def ensure_spans_flushed():
20142014
if inspect.isfunction(data_iterator) or inspect.isroutine(data_iterator):
20152015
data_iterator = data_iterator()
20162016

2017+
if inspect.isawaitable(data_iterator):
2018+
data_iterator = await data_iterator
2019+
20172020
if not inspect.isasyncgen(data_iterator):
20182021

20192022
async def to_async(it):

‎py/src/braintrust/test_framework.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,46 @@ def exact_match(input_value, output, expected):
287287
assert result.summary.scores["exact_match"].score == 1.0
288288

289289

290+
@pytest.mark.parametrize("is_async", [False, True], ids=["Eval", "EvalAsync"])
291+
@pytest.mark.parametrize("data_kind", ["iterator", "factory", "awaitable", "async-factory"])
292+
def test_eval_resolves_awaitable_data(is_async, data_kind):
293+
cases = [EvalCase(input=1, expected=2), EvalCase(input=2, expected=4)]
294+
295+
async def load_data():
296+
await asyncio.sleep(0)
297+
return iter(cases)
298+
299+
if data_kind == "iterator":
300+
data = iter(cases)
301+
elif data_kind == "factory":
302+
data = lambda: iter(cases)
303+
elif data_kind == "awaitable":
304+
data = load_data()
305+
else:
306+
data = load_data
307+
308+
def exact_match(output, expected):
309+
return float(output == expected)
310+
311+
evaluate = EvalAsync if is_async else Eval
312+
result = evaluate(
313+
"test-awaitable-data",
314+
data=data,
315+
task=lambda input_value: input_value * 2,
316+
scores=[exact_match],
317+
no_send_logs=True,
318+
enable_cache=False,
319+
)
320+
if is_async:
321+
result = asyncio.run(result)
322+
323+
assert [(row.input, row.output, row.scores, row.error) for row in result.results] == [
324+
(1, 2, {"exact_match": 1.0}, None),
325+
(2, 4, {"exact_match": 1.0}, None),
326+
]
327+
assert result.summary.scores["exact_match"].score == 1.0
328+
329+
290330
@pytest.mark.parametrize("is_async", [False, True], ids=["Eval", "EvalAsync"])
291331
@pytest.mark.parametrize("outcome", ["task_error", "scorer_error", "success"])
292332
@pytest.mark.parametrize("handler_result", ["score", "none", "default"])

0 commit comments

Comments
 (0)