Skip to content

Commit 46db9af

Browse files
committed
fix(experiments): await composite evaluator results
Address review feedback on langfuse#1869 and preserve composite scores from Future, Task and custom awaitable callbacks. Note: pre-existing failures in prompt, prompt atexit, and Windows path tests are not addressed by this PR.
1 parent e174598 commit 46db9af

2 files changed

Lines changed: 34 additions & 1 deletion

File tree

‎langfuse/_client/client.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"""
55

66
import asyncio
7+
import inspect
78
import logging
89
import os
910
import re
@@ -3183,7 +3184,7 @@ async def _process_experiment_item(
31833184
evaluations=evaluations,
31843185
)
31853186

3186-
if asyncio.iscoroutine(result):
3187+
if inspect.isawaitable(result):
31873188
result = await result
31883189

31893190
composite_evals = _normalize_evaluator_result(

‎tests/unit/test_experiment.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,38 @@ def evaluator(**kwargs):
7979
]
8080
assert [(e.name, e.value) for e in result.run_evaluations] == [("quality", 1.0)]
8181

82+
def test_composite_evaluation_is_resolved_and_scored(
83+
self, langfuse_memory_client, wrap_result, monkeypatch
84+
):
85+
create_score = MagicMock()
86+
monkeypatch.setattr(langfuse_memory_client, "create_score", create_score)
87+
88+
def task(*, item):
89+
return "answer"
90+
91+
def evaluator(**kwargs):
92+
return {"name": "quality", "value": 1.0}
93+
94+
def composite_evaluator(*, evaluations, **kwargs):
95+
return wrap_result({"name": "aggregate", "value": evaluations[0].value})
96+
97+
result = langfuse_memory_client.run_experiment(
98+
name="awaitable-composite",
99+
data=[{"input": "question"}],
100+
task=task,
101+
evaluators=[evaluator],
102+
composite_evaluator=composite_evaluator,
103+
)
104+
105+
assert [(e.name, e.value) for e in result.item_results[0].evaluations] == [
106+
("quality", 1.0),
107+
("aggregate", 1.0),
108+
]
109+
assert [
110+
(call.kwargs["name"], call.kwargs["value"])
111+
for call in create_score.call_args_list
112+
] == [("quality", 1.0), ("aggregate", 1.0)]
113+
82114

83115
def _noop_task(*, item, **kwargs): # pragma: no cover - never invoked via mock
84116
return None

0 commit comments

Comments
 (0)