Skip to content

Commit 7f7b714

Browse files
fix: address v5 release review findings (#1993)
* fix: budget propagated trace metadata against the span attribute limit Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> * fix: gate experiment run scores on the item span's sampling decision Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> * fix(batch-evaluation): skip boundary duplicates on resume and fetch updatedAt for dedup Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> * style: format span metadata key helper Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> * fix: count experiment items as sampled only after experiment attributes are set Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> * fix(batch-evaluation): request updatedAt when no field groups are given Co-authored-by: hassieb.pakzad <hassieb.pakzad@clickhouse.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
1 parent c92a1ca commit 7f7b714

6 files changed

Lines changed: 197 additions & 50 deletions

File tree

‎langfuse/_client/client.py‎

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"""
55

66
import asyncio
7+
import json
78
import logging
89
import os
910
import re
@@ -19,6 +20,7 @@
1920
List,
2021
Literal,
2122
Optional,
23+
Set,
2224
Tuple,
2325
Type,
2426
Union,
@@ -110,6 +112,7 @@
110112
Prompt_Chat,
111113
Prompt_Text,
112114
)
115+
from langfuse.api.core.datetime_utils import serialize_datetime
113116
from langfuse.batch_evaluation import (
114117
BatchEvaluationResult,
115118
BatchEvaluationResumeToken,
@@ -2673,6 +2676,7 @@ async def _run_experiment_async(
26732676

26742677
# Set up concurrency control
26752678
semaphore = asyncio.Semaphore(max_concurrency)
2679+
sampled_trace_ids: Set[str] = set()
26762680

26772681
# Process all items
26782682
async def process_item(item: ExperimentItem) -> ExperimentItemResult:
@@ -2688,6 +2692,7 @@ async def process_item(item: ExperimentItem) -> ExperimentItemResult:
26882692
description,
26892693
metadata,
26902694
experiment_item_version,
2695+
sampled_trace_ids,
26912696
)
26922697

26932698
# Run all items concurrently
@@ -2721,14 +2726,7 @@ async def process_item(item: ExperimentItem) -> ExperimentItemResult:
27212726

27222727
# Without an exported item span the experiment does not exist on the server,
27232728
# so a run score would be orphaned.
2724-
stored_run_evaluations = (
2725-
run_evaluations
2726-
if any(
2727-
result.trace_id is not None and self._is_trace_sampled(result.trace_id)
2728-
for result in valid_results
2729-
)
2730-
else []
2731-
)
2729+
stored_run_evaluations = run_evaluations if sampled_trace_ids else []
27322730
if run_evaluations and not stored_run_evaluations:
27332731
langfuse_logger.debug(
27342732
"Skipping run-level scores for experiment %s: all items were sampled out.",
@@ -2782,8 +2780,6 @@ def _create_experiment_id(
27822780
)
27832781
return self._create_observation_id()
27842782

2785-
import json
2786-
27872783
# Must match the Langfuse server's stable experiment id derivation byte for
27882784
# byte (JS JSON.stringify of the same array, SHA-256, first 16 hex chars).
27892785
payload = json.dumps(
@@ -2794,27 +2790,8 @@ def _create_experiment_id(
27942790

27952791
return sha256(payload.encode("utf-8")).hexdigest()[:16]
27962792

2797-
def _is_trace_sampled(self, trace_id: str) -> bool:
2798-
from opentelemetry.sdk.trace.sampling import Decision
2799-
2800-
tracer_provider = self._resources.tracer_provider if self._resources else None
2801-
sampler = getattr(tracer_provider, "sampler", None)
2802-
if sampler is None:
2803-
return True
2804-
2805-
try:
2806-
decision = sampler.should_sample(
2807-
parent_context=None, trace_id=int(trace_id, 16), name="experiment"
2808-
).decision
2809-
except Exception:
2810-
return True
2811-
2812-
return bool(decision == Decision.RECORD_AND_SAMPLE)
2813-
28142793
@staticmethod
28152794
def _format_experiment_item_version(dataset_version: datetime) -> str:
2816-
from langfuse.api.core.datetime_utils import serialize_datetime
2817-
28182795
return serialize_datetime(dataset_version)
28192796

28202797
async def _process_experiment_item(
@@ -2829,6 +2806,7 @@ async def _process_experiment_item(
28292806
experiment_description: Optional[str],
28302807
experiment_metadata: Optional[Dict[str, Any]] = None,
28312808
experiment_item_version: Optional[str] = None,
2809+
sampled_trace_ids: Optional[Set[str]] = None,
28322810
) -> ExperimentItemResult:
28332811
with self.start_as_current_observation(name="experiment-item-run") as span:
28342812
try:
@@ -2937,6 +2915,13 @@ async def _process_experiment_item(
29372915
otel_context_api.get_current()
29382916
),
29392917
)
2918+
# Only a sampled span carrying the experiment attributes
2919+
# creates the experiment on the server.
2920+
if (
2921+
sampled_trace_ids is not None
2922+
and span._otel_span.get_span_context().trace_flags.sampled
2923+
):
2924+
sampled_trace_ids.add(span.trace_id)
29402925
# Write the observation metadata after the experiment
29412926
# attributes, so the span attribute limit trims the
29422927
# metadata instead of leaving no room for the output.

‎langfuse/_client/span.py‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,17 @@
6464
_OBSERVATION_CLASS_MAP: Dict[str, Type["LangfuseObservationWrapper"]] = {}
6565

6666

67-
def _is_observation_metadata_key(key: str) -> bool:
68-
prefix = LangfuseOtelSpanAttributes.OBSERVATION_METADATA
69-
return key == prefix or key.startswith(prefix + ".")
67+
_METADATA_KEY_PREFIXES: Tuple[str, ...] = (
68+
LangfuseOtelSpanAttributes.OBSERVATION_METADATA,
69+
LangfuseOtelSpanAttributes.TRACE_METADATA,
70+
)
71+
72+
73+
def _is_metadata_key(key: str) -> bool:
74+
return any(
75+
key == prefix or key.startswith(prefix + ".")
76+
for prefix in _METADATA_KEY_PREFIXES
77+
)
7078

7179

7280
# Attributes the SDK may write in later updates of an observation. Metadata
@@ -151,9 +159,7 @@ def _drop_attributes_over_span_limit(
151159
if len(existing) + reserved_count + len(new_keys) <= max_attributes:
152160
return attributes
153161

154-
new_metadata_keys = [
155-
key for key in new_keys if _is_observation_metadata_key(key)
156-
]
162+
new_metadata_keys = [key for key in new_keys if _is_metadata_key(key)]
157163
free_metadata_slots = max(
158164
0,
159165
max_attributes
@@ -171,12 +177,8 @@ def _drop_attributes_over_span_limit(
171177
dropped_other = remaining_new_keys[capacity:]
172178
# The metadata budget is never looser than the hard guard, so only
173179
# non-metadata keys can be dropped here. Count any metadata defensively.
174-
dropped_metadata += [
175-
key for key in dropped_other if _is_observation_metadata_key(key)
176-
]
177-
dropped_other = [
178-
key for key in dropped_other if not _is_observation_metadata_key(key)
179-
]
180+
dropped_metadata += [key for key in dropped_other if _is_metadata_key(key)]
181+
dropped_other = [key for key in dropped_other if not _is_metadata_key(key)]
180182

181183
if not dropped_metadata and not dropped_other:
182184
return attributes

‎langfuse/batch_evaluation.py‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@
3939
"""
4040

4141

42+
def _with_time_fields(fields: Optional[str]) -> str:
43+
"""Add the ``time`` group, whose ``updatedAt`` picks the newest duplicate row."""
44+
groups = [group.strip() for group in (fields or "").split(",") if group.strip()]
45+
if not groups:
46+
# The API's default when no groups are requested.
47+
groups = ["core", "basic"]
48+
if "time" not in groups:
49+
groups.append("time")
50+
return ",".join(groups)
51+
52+
4253
class EvaluatorInputs:
4354
"""Input data structure for evaluators, returned by mapper functions.
4455
@@ -469,6 +480,9 @@ class BatchEvaluationResumeToken:
469480
observation. Only used to resume when `cursor` is None, by fetching
470481
observations that started strictly before this timestamp.
471482
last_processed_id: The ID of the last processed observation, for reference.
483+
last_page_ids: IDs of the observations on the last processed page. A
484+
resumed run skips these, because unmerged duplicate rows of one
485+
observation can span a page boundary.
472486
items_processed: Number of items successfully processed so far, including
473487
items processed by the runs this token was resumed from.
474488
@@ -515,6 +529,7 @@ def __init__(
515529
last_processed_id: str,
516530
items_processed: int,
517531
cursor: Optional[str] = None,
532+
last_page_ids: Optional[List[str]] = None,
518533
):
519534
"""Initialize BatchEvaluationResumeToken with the provided state.
520535
@@ -524,6 +539,7 @@ def __init__(
524539
last_processed_id: ID of last processed item.
525540
items_processed: Count of items processed before interruption.
526541
cursor: Cursor of the next page to fetch.
542+
last_page_ids: IDs of the observations on the last processed page.
527543
528544
Note:
529545
All arguments must be provided as keywords.
@@ -533,6 +549,7 @@ def __init__(
533549
self.last_processed_timestamp = last_processed_timestamp
534550
self.last_processed_id = last_processed_id
535551
self.items_processed = items_processed
552+
self.last_page_ids = last_page_ids
536553

537554

538555
class BatchEvaluationResult:
@@ -896,7 +913,9 @@ async def run_async(
896913
and resume_from.last_processed_timestamp
897914
else None
898915
)
899-
previous_page_ids: Set[Optional[str]] = set()
916+
previous_page_ids: Set[Optional[str]] = set(
917+
(resume_from.last_page_ids or []) if resume_from else []
918+
)
900919
batch_number = 0
901920

902921
if verbose:
@@ -917,6 +936,9 @@ def build_resume_token() -> BatchEvaluationResumeToken:
917936
last_processed_timestamp=last_item_timestamp,
918937
last_processed_id=last_item_id,
919938
items_processed=previously_processed + total_items_processed,
939+
last_page_ids=sorted(
940+
item_id for item_id in previous_page_ids if item_id is not None
941+
),
920942
)
921943

922944
while True:
@@ -1091,7 +1113,7 @@ async def _fetch_batch_with_retry(
10911113
"""
10921114
response = await asyncio.to_thread(
10931115
self.client.api.observations.get_many,
1094-
fields=fields,
1116+
fields=_with_time_fields(fields),
10951117
limit=limit,
10961118
cursor=cursor,
10971119
filter=filter,

‎tests/unit/test_batch_evaluation.py‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,8 @@ async def test_paginates_with_cursor_and_scores_observations():
117117
assert [call["cursor"] for call in api.calls] == [None, "2", "4"]
118118
assert all("page" not in call for call in api.calls)
119119
assert all(
120-
call["fields"] == DEFAULT_BATCH_EVALUATION_FIELDS and call["filter"] is None
120+
call["fields"] == f"{DEFAULT_BATCH_EVALUATION_FIELDS},time"
121+
and call["filter"] is None
121122
for call in api.calls
122123
)
123124
assert result.completed is True
@@ -151,11 +152,21 @@ def recording_mapper(*, item):
151152

152153
await run(runner, mapper=recording_mapper, fields="core,io")
153154

154-
assert api.calls[0]["fields"] == "core,io"
155+
assert api.calls[0]["fields"] == "core,io,time"
155156
assert isinstance(seen[0], ObservationV2)
156157
assert seen[0].input == '{"question": 1}'
157158

158159

160+
@pytest.mark.asyncio
161+
@pytest.mark.parametrize("fields", [None, ""])
162+
async def test_unset_fields_request_the_api_defaults_plus_time(fields):
163+
runner, api, _ = make_runner([make_observation(1)])
164+
165+
await run(runner, fields=fields)
166+
167+
assert api.calls[0]["fields"] == "core,basic,time"
168+
169+
159170
def test_client_and_runner_share_default_fields():
160171
client_default = (
161172
inspect.signature(Langfuse.run_batched_evaluation).parameters["fields"].default
@@ -378,3 +389,24 @@ def recording_mapper(*, item):
378389
assert [item.output for item in seen if item.id == "obs-1"] == ["fresh answer"]
379390
assert result.total_items_processed == 2
380391
assert client.create_score.call_count == 2
392+
393+
394+
@pytest.mark.asyncio
395+
async def test_resume_skips_duplicate_rows_split_across_the_page_boundary():
396+
duplicate = make_observation(2).model_copy(
397+
update={"updated_at": BASE_TIME + timedelta(hours=1)}
398+
)
399+
runner, _, client = make_runner(
400+
[make_observation(i) for i in range(5)] + [duplicate]
401+
)
402+
403+
first = await run(runner, max_items=3, fetch_batch_size=3)
404+
assert first.resume_token is not None
405+
assert first.resume_token.last_page_ids == ["obs-2", "obs-3", "obs-4"]
406+
407+
await run(runner, resume_from=first.resume_token, fetch_batch_size=3)
408+
409+
scored_ids = [
410+
call.kwargs["observation_id"] for call in client.create_score.call_args_list
411+
]
412+
assert sorted(scored_ids) == [f"obs-{i}" for i in range(5)]

‎tests/unit/test_experiment.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -768,6 +768,80 @@ def test_run_scores_skipped_when_all_items_are_sampled_out(
768768
assert [e.name for e in result.run_evaluations] == ["run"]
769769
assert _run_scores(create_score) == []
770770

771+
def test_run_scores_follow_the_item_span_sampling_decision(
772+
self, langfuse_memory_client, monkeypatch
773+
):
774+
from opentelemetry.sdk.trace.sampling import (
775+
Decision,
776+
Sampler,
777+
SamplingResult,
778+
)
779+
780+
class ItemSpanSampler(Sampler):
781+
def should_sample(self, parent_context, trace_id, name, *args, **kwargs):
782+
decision = (
783+
Decision.RECORD_AND_SAMPLE
784+
if name.startswith("experiment-item")
785+
else Decision.DROP
786+
)
787+
return SamplingResult(decision)
788+
789+
def get_description(self):
790+
return "ItemSpanSampler"
791+
792+
sampler = ItemSpanSampler()
793+
create_score = MagicMock()
794+
monkeypatch.setattr(langfuse_memory_client, "create_score", create_score)
795+
monkeypatch.setattr(langfuse_memory_client, "_get_project_id", lambda: "p")
796+
monkeypatch.setattr(
797+
langfuse_memory_client._resources.tracer_provider, "sampler", sampler
798+
)
799+
monkeypatch.setattr(langfuse_memory_client._otel_tracer, "sampler", sampler)
800+
801+
langfuse_memory_client.run_experiment(
802+
name="exp",
803+
data=[{"input": "a"}],
804+
task=lambda *, item, **kwargs: item["input"],
805+
run_evaluators=[lambda **kwargs: Evaluation(name="run", value=1.0)],
806+
)
807+
808+
assert len(_run_scores(create_score)) == 1
809+
810+
def test_run_scores_skipped_when_all_items_fail_before_experiment_attributes(
811+
self, langfuse_memory_client, monkeypatch
812+
):
813+
create_score = MagicMock()
814+
monkeypatch.setattr(langfuse_memory_client, "create_score", create_score)
815+
monkeypatch.setattr(langfuse_memory_client, "_get_project_id", lambda: "p")
816+
817+
langfuse_memory_client.run_experiment(
818+
name="exp",
819+
data=[{"input": None}],
820+
task=lambda *, item, **kwargs: item["input"],
821+
run_evaluators=[lambda **kwargs: Evaluation(name="run", value=1.0)],
822+
)
823+
824+
assert _run_scores(create_score) == []
825+
826+
def test_run_scores_kept_when_the_task_fails_after_experiment_attributes(
827+
self, langfuse_memory_client, monkeypatch
828+
):
829+
def failing_task(*, item, **kwargs):
830+
raise RuntimeError("boom")
831+
832+
create_score = MagicMock()
833+
monkeypatch.setattr(langfuse_memory_client, "create_score", create_score)
834+
monkeypatch.setattr(langfuse_memory_client, "_get_project_id", lambda: "p")
835+
836+
langfuse_memory_client.run_experiment(
837+
name="exp",
838+
data=[{"input": "a"}],
839+
task=failing_task,
840+
run_evaluators=[lambda **kwargs: Evaluation(name="run", value=1.0)],
841+
)
842+
843+
assert len(_run_scores(create_score)) == 1
844+
771845
def test_tracing_disabled_skips_project_id_lookup(self, monkeypatch):
772846
client = Langfuse(
773847
public_key="pk", secret_key="sk", base_url="http://x", tracing_enabled=False

0 commit comments

Comments
 (0)