fix(sdk): normalize failed experiment scores

Author: aiedwardyi

sdks/python/src/opik/evaluation/evaluation_result.py

@@ -85,6 +85,19 @@ def normalize_experiment_score(
             metadata={**(base_metadata or {}), "_fabricated": True},
         )
 
+    if score.metadata is not None and not isinstance(score.metadata, Mapping):
+        return score_result.ScoreResult(
+            name=effective_name,
+            value=0.0,
+            reason=(
+                "ScoreResult.metadata must be a mapping or None, got "
+                f"{type(score.metadata).__name__}."
+            ),
+            scoring_failed=True,
+            category_name=score.category_name,
+            metadata={"_fabricated": True},
+        )
+
     if score.scoring_failed:
         value = (
             score.value

sdks/python/src/opik/evaluation/evaluator.py

@@ -65,7 +65,7 @@ def _deduplicate_experiment_scores(
     scores: List[object],
 ) -> List[score_result.ScoreResult]:
     deduplicated: List[score_result.ScoreResult] = []
-    successful_positions: Dict[str, int] = {}
+    score_positions: Dict[str, int] = {}
     for raw_score in scores:
         if not isinstance(raw_score, score_result.ScoreResult):
             LOGGER.warning(
@@ -75,12 +75,9 @@ def _deduplicate_experiment_scores(
         score = evaluation_result.normalize_experiment_score(
             raw_score, default_name="invalid_experiment_score"
         )
-        if score.scoring_failed:
-            deduplicated.append(score)
-            continue
-        position = successful_positions.get(score.name)
+        position = score_positions.get(score.name)
         if position is None:
-            successful_positions[score.name] = len(deduplicated)
+            score_positions[score.name] = len(deduplicated)
             deduplicated.append(score)
         else:
             deduplicated[position] = score

sdks/python/tests/unit/evaluation/test_evaluate_experiment.py

@@ -5,7 +5,12 @@
 
 from opik import evaluation, exceptions, url_helpers
 from opik.api_objects import opik_client
-from opik.evaluation import metrics, rest_operations, test_case
+from opik.evaluation import (
+    evaluator as evaluator_module,
+    metrics,
+    rest_operations,
+    test_case,
+)
 from opik.evaluation.engine import engine
 from opik.evaluation.metrics import score_result
 
@@ -348,6 +353,33 @@ def scorer_b(_):
     )
 
 
+@pytest.mark.parametrize(
+    "scores, expected",
+    [
+        (
+            [
+                score_result.ScoreResult(name="accuracy", value=0.5),
+                score_result.ScoreResult(
+                    name="accuracy", value=0.0, scoring_failed=True
+                ),
+            ],
+            [score_result.ScoreResult(name="accuracy", value=0.0, scoring_failed=True)],
+        ),
+        (
+            [
+                score_result.ScoreResult(
+                    name="accuracy", value=0.0, scoring_failed=True
+                ),
+                score_result.ScoreResult(name="accuracy", value=0.9),
+            ],
+            [score_result.ScoreResult(name="accuracy", value=0.9)],
+        ),
+    ],
+)
+def test_deduplicate_experiment_scores__last_result_wins(scores, expected):
+    assert evaluator_module._deduplicate_experiment_scores(scores) == expected
+
+
 def test_evaluate_experiment__with_experiment_id__uses_get_by_id(fake_backend):
     mock_experiment = _make_mock_experiment(id="explicit-exp-id")
     mock_dataset = _make_mock_dataset()

sdks/python/tests/unit/evaluation/test_evaluation_result.py

@@ -796,6 +796,21 @@ def test_normalize_experiment_score__invalid_metadata_type(bad_metadata):
     assert res.metadata == {"_fabricated": True}
 
 
+@pytest.mark.parametrize("bad_metadata", ["invalid_str", [1, 2], 123, ("a", "b")])
+def test_normalize_experiment_score__valid_score_with_invalid_metadata(bad_metadata):
+    score = score_result.ScoreResult(
+        name="accuracy",
+        value=0.5,
+        metadata=bad_metadata,  # type: ignore
+    )
+    res = evaluation_result.normalize_experiment_score(score, default_name="fallback")
+    assert res.name == "accuracy"
+    assert res.value == 0.0
+    assert res.scoring_failed is True
+    assert res.metadata == {"_fabricated": True}
+    assert "metadata must be a mapping" in res.reason
+
+
 @pytest.mark.parametrize("bad_flag", ["false", "true", 0, 1, None, []])
 def test_normalize_experiment_score__invalid_scoring_failed_flag(bad_flag):
     score = score_result.ScoreResult(