Skip to content

Commit 590b0a9

Browse files
fix(batch_evaluation): route batch_evaluation through v2 observations API
BatchEvaluationRunner._fetch_batch_with_retry used the v3 read APIs (GET /api/public/traces and GET /api/public/observations via legacy.observations_v1) to fetch items for batch evaluation. Both endpoints return HTTP 400 on Langfuse platform v4 events_only deployments, so every batch evaluation against an event-store-backed self-hosted Langfuse v4 fails on the first page (langfuse/langfuse#1861). This change routes the runner through GET /api/public/v2/observations, which is the only read path that works on v4 events_only (and remains available on v3, with the v3 endpoint scheduled for removal on 2026-11-16). The pagination state switches from page-based to cursor-based. For scope=traces, observations are collapsed to one representative per trace (preferring is_root_observation=True), since the v2 endpoint has no trace-level read. The MapperFunction protocol accepts the v2 ObservationV2 in addition to the legacy TraceWithFullDetails / ObservationsView, so existing mappers keep working with the input/output/metadata fields they already read.
1 parent 65392c7 commit 590b0a9

2 files changed

Lines changed: 345 additions & 51 deletions

File tree

‎langfuse/batch_evaluation.py‎

Lines changed: 138 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,65 @@
2525

2626
from langfuse.api import (
2727
ObservationsView,
28+
ObservationV2,
2829
TraceWithFullDetails,
2930
)
3031
from langfuse.experiment import Evaluation, EvaluatorFunction
3132
from langfuse.logger import langfuse_logger as logger
3233

34+
# v2 observations field groups. ``core`` and ``basic`` carry id/trace_id
35+
# and the typical observation metadata; ``io`` adds the input/output/metadata
36+
# the mapper reads, ``usage`` carries cost and token details, ``model``
37+
# surfaces the model id, ``trace_context`` exposes tags/release/trace_name.
38+
# The legacy ``/api/public/traces`` ``io`` / ``scores`` / ``observations`` /
39+
# ``metrics`` field groups are not selectable on the v2 read endpoint because
40+
# it returns one observation at a time.
41+
_DEFAULT_V2_OBSERVATION_FIELDS = "core,basic,io,usage,model,trace_context"
42+
_LEGACY_ONLY_FIELDS = frozenset({"observations", "scores"})
43+
44+
45+
def _v2_observations_fields(fetch_fields: Optional[str]) -> str:
46+
"""Translate the legacy ``fetch_trace_fields`` argument into the v2 field
47+
groups understood by ``GET /api/public/v2/observations``.
48+
49+
The default groups cover everything the existing evaluator contract reads.
50+
Caller-supplied groups are merged in; legacy-only groups are dropped.
51+
"""
52+
if fetch_fields is None:
53+
return _DEFAULT_V2_OBSERVATION_FIELDS
54+
55+
user_groups = {item.strip() for item in fetch_fields.split(",") if item.strip()}
56+
base_groups = {item.strip() for item in _DEFAULT_V2_OBSERVATION_FIELDS.split(",")}
57+
merged = (user_groups | base_groups) - _LEGACY_ONLY_FIELDS
58+
if not merged:
59+
return _DEFAULT_V2_OBSERVATION_FIELDS
60+
return ",".join(sorted(merged))
61+
62+
63+
def _collapse_observations_to_traces(
64+
observations: List[ObservationV2],
65+
) -> List[ObservationV2]:
66+
"""Collapse a flat list of observations to one observation per trace_id.
67+
68+
The v2 observations endpoint has no trace-level read; this helper takes
69+
whatever observations the page returned and returns one representative
70+
per trace, preferring the observation that the server already marked as
71+
the root (``is_root_observation=True``) and falling back to the first
72+
observation seen for that trace.
73+
"""
74+
seen: Dict[str, ObservationV2] = {}
75+
preferred: Dict[str, ObservationV2] = {}
76+
for observation in observations:
77+
trace_id = getattr(observation, "trace_id", None)
78+
if not trace_id or trace_id in seen:
79+
continue
80+
seen[trace_id] = observation
81+
if getattr(observation, "is_root_observation", False):
82+
preferred[trace_id] = observation
83+
84+
return [preferred.get(trace_id, fallback) for trace_id, fallback in seen.items()]
85+
86+
3387
if TYPE_CHECKING:
3488
from langfuse._client.client import Langfuse
3589

@@ -137,7 +191,7 @@ class MapperFunction(Protocol):
137191
def __call__(
138192
self,
139193
*,
140-
item: Union["TraceWithFullDetails", "ObservationsView"],
194+
item: Union["TraceWithFullDetails", "ObservationsView", "ObservationV2"],
141195
**kwargs: Dict[str, Any],
142196
) -> Union[EvaluatorInputs, Awaitable[EvaluatorInputs]]:
143197
"""Transform an API response object into evaluator inputs.
@@ -148,8 +202,11 @@ def __call__(
148202
149203
Args:
150204
item: The API response object to transform. The type depends on the scope:
151-
- TraceWithFullDetails: When evaluating traces
152-
- ObservationsView: When evaluating observations
205+
- TraceWithFullDetails: legacy v3 trace response
206+
- ObservationsView: legacy v1 observations response
207+
- ObservationV2: v2 observations response (used on Langfuse
208+
platform v4 events_only deployments where the legacy endpoints
209+
are unavailable)
153210
154211
Returns:
155212
EvaluatorInputs: A structured container with:
@@ -920,8 +977,10 @@ async def run_async(
920977
# Create semaphore for concurrency control
921978
semaphore = asyncio.Semaphore(max_concurrency)
922979

923-
# Pagination state
924-
page = 1
980+
# Pagination state. The v2 observations endpoint is cursor-based, so
981+
# the runner walks the result set with ``next_cursor`` and stops when
982+
# the server returns ``meta.cursor=None``.
983+
cursor: Optional[str] = None
925984
has_more = True
926985
last_item_timestamp: Optional[str] = None
927986
last_item_id: Optional[str] = None
@@ -948,10 +1007,10 @@ async def run_async(
9481007

9491008
# Fetch next batch with retry logic
9501009
try:
951-
items = await self._fetch_batch_with_retry(
1010+
items, next_cursor = await self._fetch_batch_with_retry(
9521011
scope=scope,
9531012
filter=effective_filter,
954-
page=page,
1013+
cursor=cursor,
9551014
limit=fetch_batch_size,
9561015
max_retries=max_retries,
9571016
fields=fetch_trace_fields,
@@ -986,17 +1045,23 @@ async def run_async(
9861045
item_evaluations=item_evaluations,
9871046
)
9881047

1048+
# Advance the cursor. ``None`` means the server has no further
1049+
# pages; preserve the cursor on empty responses so a transient
1050+
# empty page does not silently stop iteration.
1051+
cursor = next_cursor
1052+
if cursor is None:
1053+
has_more = False
1054+
9891055
# Check if we got any items
9901056
if not items:
991-
has_more = False
9921057
if verbose:
9931058
logger.info("No more items to fetch")
9941059
break
9951060

9961061
total_items_fetched += len(items)
9971062

9981063
if verbose:
999-
logger.info("Fetched batch %s (%s items)", page, len(items))
1064+
logger.info("Fetched batch (%s items)", len(items))
10001065

10011066
# Limit items if max_items would be exceeded
10021067
items_to_process = items
@@ -1013,7 +1078,7 @@ async def run_async(
10131078

10141079
# Process items concurrently
10151080
async def process_item(
1016-
item: Union[TraceWithFullDetails, ObservationsView],
1081+
item: Union[TraceWithFullDetails, ObservationsView, ObservationV2],
10171082
) -> Tuple[str, Union[Tuple[int, int, int, List[Evaluation]], Exception]]:
10181083
"""Process a single item and return (item_id, result)."""
10191084
async with semaphore:
@@ -1094,17 +1159,12 @@ async def process_item(
10941159
total_scores_created,
10951160
)
10961161

1097-
# Check if we should continue to next page
1098-
if len(items) < fetch_batch_size:
1099-
# Last page - no more items available
1100-
has_more = False
1101-
else:
1102-
page += 1
1103-
1104-
# Check max_items again before next fetch
1105-
if max_items is not None and total_items_fetched >= max_items:
1106-
has_more = True # More items exist but we're stopping
1107-
break
1162+
# Check if we should continue to next page. ``has_more`` is set inside
1163+
# the fetch block above, so we only need to honour ``max_items``
1164+
# here.
1165+
if max_items is not None and total_items_fetched >= max_items:
1166+
has_more = True # More items exist but we're stopping
1167+
break
11081168

11091169
# Flush all scores to Langfuse
11101170
if verbose:
@@ -1152,52 +1212,79 @@ async def _fetch_batch_with_retry(
11521212
*,
11531213
scope: str,
11541214
filter: Optional[str],
1155-
page: int,
1215+
cursor: Optional[str],
11561216
limit: int,
11571217
max_retries: int,
11581218
fields: Optional[str],
1159-
) -> List[Union[TraceWithFullDetails, ObservationsView]]:
1160-
"""Fetch a batch of items with retry logic.
1219+
) -> Tuple[
1220+
List[Union[TraceWithFullDetails, ObservationsView, ObservationV2]],
1221+
Optional[str],
1222+
]:
1223+
"""Fetch a batch of items via the v2 observations API with cursor pagination.
1224+
1225+
Both ``scope='traces'`` and ``scope='observations'`` go through the
1226+
v2 ``GET /api/public/v2/observations`` endpoint, which is the only
1227+
read path that works on Langfuse platform v4 events_only deployments.
1228+
The legacy ``GET /api/public/traces`` and ``GET /api/public/observations``
1229+
endpoints are unavailable there.
11611230
11621231
Args:
11631232
scope: The type of items ("traces", "observations").
11641233
filter: JSON filter string for querying.
1165-
page: Page number (1-indexed).
1166-
limit: Number of items per page.
1234+
cursor: Pagination cursor returned by the previous page; ``None`` on
1235+
the first call.
1236+
limit: Number of items to request per page.
11671237
max_retries: Maximum number of retry attempts.
1168-
verbose: Whether to log retry attempts.
1169-
fields: Trace fields to fetch
1238+
fields: Comma-separated list of v2 field groups to include when
1239+
fetching traces. Maps to the ``fields`` query parameter on
1240+
``/api/public/v2/observations``. Only used when
1241+
``scope='traces'``.
11701242
11711243
Returns:
1172-
List of items from the API.
1244+
Tuple of (items, next_cursor). ``items`` is a list of either
1245+
``ObservationV2`` (when scope='observations') or one
1246+
``ObservationV2`` per trace (when scope='traces'). ``next_cursor``
1247+
is the cursor for the next page, or ``None`` when no more pages
1248+
remain.
11731249
11741250
Raises:
1251+
ValueError: If ``scope`` is not "traces" or "observations".
11751252
Exception: If all retry attempts fail.
11761253
"""
1177-
if scope == "traces":
1178-
response = self.client.api.trace.list(
1179-
page=page,
1180-
limit=limit,
1181-
filter=filter,
1182-
request_options={"max_retries": max_retries},
1183-
fields=fields,
1184-
) # type: ignore
1185-
return list(response.data) # type: ignore
1186-
elif scope == "observations":
1187-
response = self.client.api.legacy.observations_v1.get_many(
1188-
page=page,
1189-
limit=limit,
1190-
filter=filter,
1191-
request_options={"max_retries": max_retries},
1192-
) # type: ignore
1193-
return list(response.data) # type: ignore
1194-
else:
1254+
if scope not in ("traces", "observations"):
11951255
error_message = f"Invalid scope: {scope}"
11961256
raise ValueError(error_message)
11971257

1258+
v2_fields = _v2_observations_fields(fields)
1259+
1260+
response = self.client.api.observations.get_many( # type: ignore[union-attr]
1261+
cursor=cursor,
1262+
limit=limit,
1263+
filter=filter,
1264+
request_options={"max_retries": max_retries},
1265+
fields=v2_fields,
1266+
)
1267+
1268+
next_cursor = cast(
1269+
Optional[str], response.meta.cursor if response.meta else None
1270+
)
1271+
1272+
if scope == "traces":
1273+
items = cast(
1274+
List[Union[TraceWithFullDetails, ObservationsView, ObservationV2]],
1275+
_collapse_observations_to_traces(list(response.data)), # type: ignore[arg-type]
1276+
)
1277+
else:
1278+
items = cast(
1279+
List[Union[TraceWithFullDetails, ObservationsView, ObservationV2]],
1280+
list(response.data), # type: ignore[arg-type]
1281+
)
1282+
1283+
return items, next_cursor
1284+
11981285
async def _process_batch_evaluation_item(
11991286
self,
1200-
item: Union[TraceWithFullDetails, ObservationsView],
1287+
item: Union[TraceWithFullDetails, ObservationsView, ObservationV2],
12011288
scope: str,
12021289
mapper: MapperFunction,
12031290
evaluators: List[EvaluatorFunction],
@@ -1352,7 +1439,7 @@ async def _run_evaluator_internal(
13521439
async def _run_mapper(
13531440
self,
13541441
mapper: MapperFunction,
1355-
item: Union[TraceWithFullDetails, ObservationsView],
1442+
item: Union[TraceWithFullDetails, ObservationsView, ObservationV2],
13561443
) -> EvaluatorInputs:
13571444
"""Run mapper function (handles both sync and async mappers).
13581445
@@ -1529,7 +1616,7 @@ def _build_timestamp_filter(
15291616

15301617
@staticmethod
15311618
def _get_item_id(
1532-
item: Union[TraceWithFullDetails, ObservationsView],
1619+
item: Union[TraceWithFullDetails, ObservationsView, ObservationV2],
15331620
scope: str,
15341621
) -> str:
15351622
"""Extract ID from item based on scope.
@@ -1545,7 +1632,7 @@ def _get_item_id(
15451632

15461633
@staticmethod
15471634
def _get_item_timestamp(
1548-
item: Union[TraceWithFullDetails, ObservationsView],
1635+
item: Union[TraceWithFullDetails, ObservationsView, ObservationV2],
15491636
scope: str,
15501637
) -> str:
15511638
"""Extract timestamp from item based on scope.

0 commit comments

Comments
 (0)