[NA] [SDK] fix: correct NLTK usage in METEOR and chrF metrics

apps/opik-documentation/documentation/fern/docs-v2/evaluation/metrics/heuristic_metrics.mdx

@@ -273,6 +273,8 @@ pip install bert-score
 
 `ChrF` computes the character n-gram F-score (`chrF` / `chrF++`). Adjust `beta`, `char_order`, and `word_order` to switch between the two variants.
 
+When several references are passed, the score is computed against each reference separately and the best (highest) score is returned.
+
 ```python
 from opik.evaluation.metrics import ChrF
 

apps/opik-documentation/documentation/fern/docs/evaluation/metrics/heuristic_metrics.mdx

@@ -274,6 +274,8 @@ pip install bert-score
 
 `ChrF` computes the character n-gram F-score (`chrF` / `chrF++`). Adjust `beta`, `char_order`, and `word_order` to switch between the two variants.
 
+When several references are passed, the score is computed against each reference separately and the best (highest) score is returned.
+
 ```python
 from opik.evaluation.metrics import ChrF
 

sdks/python/src/opik/evaluation/metrics/heuristics/chrf.py

@@ -13,7 +13,8 @@
 except ImportError:  # pragma: no cover - optional dependency
     nltk_chrf_score = None
 
-ChrFFn = Callable[[Sequence[str], Sequence[str]], float]
+# The candidate is a single string; only the references are a sequence.
+ChrFFn = Callable[[str, Sequence[str]], float]
 
 
 class ChrF(BaseMetric):
@@ -25,6 +26,12 @@ class ChrF(BaseMetric):
     is not supported by the NLTK backend; provide a custom ``chrf_fn`` to compute it.
     Scores range from `0.0` (no overlap) to `1.0` (perfect match).
 
+    When several references are supplied, the default backend scores the candidate
+    against each reference separately and returns the **best** (highest) score, the
+    standard multi-reference behaviour for chrF. NLTK's ``sentence_chrf`` accepts a
+    single reference only, so a list handed to it directly would be joined into one
+    string and score lower than the best individual match.
+
     References:
       - Popović, "chrF: character n-gram F-score for automatic MT evaluation" (WMT 2015)
         https://aclanthology.org/W15-3049/
@@ -44,7 +51,11 @@ class ChrF(BaseMetric):
         word_order: Maximum word n-gram order for chrF++. Not supported by the
             default NLTK backend; provide ``chrf_fn`` to use it.
         lowercase: Whether to lowercase candidate and references prior to scoring.
-        chrf_fn: Optional custom scoring callable for testing or offline usage.
+        chrf_fn: Optional custom scoring callable ``(candidate, references) -> float``
+            receiving the candidate string and the sequence of reference strings,
+            for testing or offline usage. Note this differs from NLTK's
+            ``sentence_chrf``, which takes a single reference — supplying it
+            directly means multi-reference scoring is left to your callable.
 
     Example:
         >>> from opik.evaluation.metrics import ChrF
@@ -85,20 +96,36 @@ def __init__(
                     " `pip install nltk` or provide `chrf_fn`."
                 )
 
-            def _compute(candidate: Sequence[str], references: Sequence[str]) -> float:
-                try:
-                    return float(
-                        nltk_chrf_score.sentence_chrf(
-                            references,
-                            candidate,
-                            max_len=self._char_order,
-                            beta=self._beta,
-                            ignore_whitespace=self._ignore_whitespace,
-                        )
+            def _score_single(candidate: str, reference: str) -> float:
+                # `sentence_chrf` has accepted these keywords since NLTK 3.4
+                # (2018-11-17), the release that added `ignore_whitespace`:
+                # https://github.com/nltk/nltk/blob/3.4/nltk/translate/chrf_score.py#L18
+                # It is unchanged in the latest release:
+                # https://github.com/nltk/nltk/blob/3.9.1/nltk/translate/chrf_score.py#L16
+                # Every NLTK release installable on the Python versions this SDK
+                # supports is newer than 3.4, so a TypeError here would signal a
+                # genuine error rather than an old NLTK. Catching it would mask
+                # that and silently drop char_order/beta/ignore_whitespace, so
+                # let it propagate.
+                return float(
+                    nltk_chrf_score.sentence_chrf(
+                        reference,
+                        candidate,
+                        max_len=self._char_order,
+                        beta=self._beta,
+                        ignore_whitespace=self._ignore_whitespace,
                     )
-                except TypeError:
-                    # Older NLTK versions expose the helper with fewer keyword arguments.
-                    return float(nltk_chrf_score.sentence_chrf(references, candidate))
+                )
+
+            def _compute(candidate: str, references: Sequence[str]) -> float:
+                # NLTK's sentence_chrf scores against a *single* reference; handing
+                # it the whole list makes NLTK join the references into one string,
+                # which drags the score down instead of rewarding the best match.
+                # Score each reference on its own and keep the highest, the standard
+                # multi-reference behaviour for chrF.
+                return max(
+                    _score_single(candidate, reference) for reference in references
+                )
 
             self._chrf_fn = _compute
 

sdks/python/src/opik/evaluation/metrics/heuristics/meteor.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable, Optional, Sequence, Union
+import re
+from typing import Any, Callable, Optional, Sequence, Tuple, Union
 
 try:
     import nltk  # type: ignore
@@ -10,6 +11,52 @@
 from opik.exceptions import MetricComputationError
 from opik.evaluation.metrics import base_metric, score_result
 
+# NLTK 3.6.5 switched `meteor_score` to pre-tokenized input; 3.6.4 and earlier
+# expect untokenized strings. Supporting both would mean branching on a release
+# from 2021, so the default backend requires the modern API and says so clearly.
+#
+# Upstream change: https://github.com/nltk/nltk/pull/2822 ("Accept pre-tokenized
+# references & hypothesis for METEOR calculation"), first shipped in 3.6.5 —
+# see https://github.com/nltk/nltk/blob/3.6.5/ChangeLog ("Version 3.6.5
+# 2021-10-11 ... METEOR evaluation now requires pre-tokenized input"). Compare
+# https://github.com/nltk/nltk/blob/3.6.4/nltk/translate/meteor_score.py#L343
+# with https://github.com/nltk/nltk/blob/3.6.5/nltk/translate/meteor_score.py#L347
+# for the signature change.
+MINIMUM_NLTK_VERSION = "3.6.5"
+MINIMUM_NLTK_VERSION_INFO = (3, 6, 5)
+
+# NLTK reports two-component versions for several real releases ("3.5", "3.4",
+# "3.3") and used prerelease suffixes in the past ("3.0a3"), so the version
+# string cannot be fed to a strict SemVer parser — it would raise ValueError on
+# exactly the old installs this guard exists to reject. Read the leading numeric
+# components instead.
+_VERSION_RE = re.compile(r"^\s*v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?P<suffix>\S*)")
+_PRERELEASE_RE = re.compile(r"^[-._]?(a|b|c|rc|alpha|beta|dev|pre)", re.IGNORECASE)
+
+
+def _is_below_minimum_nltk_version(version: str) -> bool:
+    """Whether `version` is known to predate `MINIMUM_NLTK_VERSION`.
+
+    Version strings that cannot be read at all — such as the ``"unknown (...)"``
+    fallback a broken install can report — return `False`. Refusing to build the
+    metric because a version string was unparseable would be worse than letting
+    NLTK speak for itself, and `_scorer` turns the resulting `TypeError` into an
+    actionable error anyway.
+    """
+    match = _VERSION_RE.match(version)
+    if match is None:
+        return False
+
+    parsed: Tuple[int, ...] = tuple(
+        int(part) if part else 0 for part in match.group(1, 2, 3)
+    )
+    if parsed == MINIMUM_NLTK_VERSION_INFO:
+        # A prerelease of the minimum version ("3.6.5rc1") predates its API;
+        # a post-release or local version ("3.6.5.post1", "3.6.5+local") does not.
+        return _PRERELEASE_RE.match(match.group("suffix")) is not None
+    return parsed < MINIMUM_NLTK_VERSION_INFO
+
+
 try:
     from nltk.translate import meteor_score as nltk_meteor_score
 except ImportError:  # pragma: no cover - optional dependency
@@ -33,9 +80,13 @@ class METEOR(base_metric.BaseMetric):
         https://huggingface.co/spaces/evaluate-metric/meteor
 
     Args:
-        meteor_fn: Optional callable with the same interface as
-            ``nltk.translate.meteor_score.meteor_score``. When omitted the
-            function from NLTK is used.
+        meteor_fn: Optional callable ``(references, hypothesis) -> float`` that
+            receives **untokenized** text: a sequence of reference strings and a
+            single hypothesis string. Note this deliberately differs from
+            ``nltk.translate.meteor_score.meteor_score``, which requires
+            pre-tokenized input — passing that function in directly will not
+            work. When omitted, NLTK is used through an adapter that tokenizes
+            on your behalf.
         alpha: Precision weight.
         beta: Penalty exponent.
         gamma: Fragmentation penalty weight.
@@ -65,6 +116,15 @@ def __init__(
                     " `pip install nltk` or provide `meteor_fn`."
                 )
 
+            installed_version = getattr(nltk, "__version__", "")
+            if nltk is not None and _is_below_minimum_nltk_version(installed_version):
+                raise ImportError(
+                    f"METEOR metric requires nltk >= {MINIMUM_NLTK_VERSION}, but "
+                    f"{installed_version} is installed. Earlier versions expect "
+                    "untokenized input. Upgrade via `pip install -U nltk` or supply "
+                    "`meteor_fn`."
+                )
+
             if nltk is not None and wordnet is not None:
                 try:
                     wordnet.ensure_loaded()  # type: ignore[attr-defined]
@@ -82,12 +142,34 @@ def __init__(
                         ) from download_error
 
             def _scorer(references: Sequence[str], hypothesis: str) -> float:
+                # NLTK's meteor_score expects pre-tokenized input: an iterable of
+                # token lists for the references and a token list for the
+                # hypothesis. Handing it raw strings raises TypeError, so tokenize
+                # here (whitespace split, matching BLEU/GLEU) while keeping the
+                # public `meteor_fn` contract string-based.
+                tokenized_references = [reference.split() for reference in references]
+                tokenized_hypothesis = hypothesis.split()
                 try:
                     return float(
                         nltk_meteor_score.meteor_score(
-                            references, hypothesis, alpha=alpha, beta=beta, gamma=gamma
+                            tokenized_references,
+                            tokenized_hypothesis,
+                            alpha=alpha,
+                            beta=beta,
+                            gamma=gamma,
                         )
                     )
+                except TypeError as error:
+                    # Only reachable when the version guard could not read
+                    # `nltk.__version__`: NLTK < 3.6.5 rejects the tokenized
+                    # input built above. Keep NLTK's own message so a genuine
+                    # type error is not misreported as a version problem.
+                    raise MetricComputationError(
+                        f"NLTK rejected the pre-tokenized METEOR input: {error}. "
+                        f"This usually means nltk < {MINIMUM_NLTK_VERSION} is "
+                        "installed, which expects untokenized strings. Upgrade via "
+                        "`pip install -U nltk` or supply `meteor_fn`."
+                    ) from error
                 except LookupError as error:
                     raise MetricComputationError(
                         "NLTK resource requirement for METEOR not satisfied. "