fix: localize repeated LLM spans separately (#86)

Reserve each verbatim answer occurrence before applying judge filters so repeated span strings keep distinct offsets and metadata. Add direct and token-level regressions for duplicate, filtered, overlapping, and excess items.

Co-authored-by: Kovács Ádám <adaam.ko@gmail.com>

Author: Sanjays2402

lettucedetect/detectors/llm.py

@@ -288,24 +288,32 @@ def _to_spans(items: list[str | dict], answer: str, min_confidence: float = 0.0)
         :returns: List of spans.
         """
         spans = []
+        used: list[tuple[int, int]] = []
         for item in items:
             if not isinstance(item, (str, dict)):
                 logger.warning("Skipping malformed hallucination item: %r", item)
                 continue
             sub = item.get("text") if isinstance(item, dict) else item
             if not sub:
                 continue
+            match = next(
+                (
+                    match
+                    for match in re.finditer(re.escape(sub), answer)
+                    if not any(match.start() < end and start < match.end() for start, end in used)
+                ),
+                None,
+            )
+            if not match:
+                logger.debug("Dropping span without an unused verbatim match in answer: %r", sub)
+                continue
+            used.append((match.start(), match.end()))
             if isinstance(item, dict):
                 if item.get("is_hallucination") is False:
                     continue
                 confidence = item.get("confidence")
                 if confidence is not None and confidence < min_confidence:
                     continue
-            # Use regex for more reliable matching
-            match = re.search(re.escape(sub), answer)
-            if not match:
-                logger.debug("Dropping span not found verbatim in answer: %r", sub)
-                continue
             span = {"start": match.start(), "end": match.end(), "text": sub}
             if isinstance(item, dict):
                 for key in ("confidence", "reasoning", "category", "subcategory"):

lettucedetect/prompts/hallucination_detection.txt

@@ -23,6 +23,7 @@ A substring that:
 ## How to choose spans
 - Flag the **minimal** substring carrying the unsupported information; do not pad with surrounding supported text.
 - Copy each span **verbatim** from the answer — character for character, including punctuation, casing, and spacing. Do not paraphrase, translate, fix typos, or normalize. A span that is not an exact substring of the answer is discarded.
+- Return spans in their left-to-right order in the answer, with at most one item for each distinct occurrence.
 - If one sentence mixes supported and unsupported content, flag only the unsupported part.
 
 ## Steps

tests/test_llm_detector_pytest.py

@@ -148,3 +148,50 @@ def test_spans_output_unchanged(self, cache_file):
                 "text": "Spain",
             }
         ]
+
+
+class TestRepeatedSpanLocalization:
+    """Repeated judge spans must map to distinct answer occurrences."""
+
+    def test_repeated_spans_keep_distinct_offsets_and_metadata(self):
+        """Each repeated item keeps its own occurrence and metadata."""
+        answer = "Paris is mentioned; Paris is repeated."
+        items: list[str | dict] = [
+            {"text": "Paris", "confidence": 0.7, "category": "first"},
+            {"text": "Paris", "confidence": 0.9, "category": "second"},
+        ]
+
+        spans = LLMDetector._to_spans(items, answer)
+
+        assert [(span["start"], span["category"]) for span in spans] == [
+            (0, "first"),
+            (20, "second"),
+        ]
+        assert [span["confidence"] for span in spans] == [0.7, 0.9]
+
+    def test_filtered_item_still_reserves_its_occurrence(self):
+        """Filtering an item must not shift a later item's location."""
+        answer = "Paris is mentioned; Paris is repeated."
+        items: list[str | dict] = [
+            {"text": "Paris", "is_hallucination": False},
+            {"text": "Paris", "is_hallucination": True},
+        ]
+
+        spans = LLMDetector._to_spans(items, answer)
+
+        assert [(span["start"], span["end"]) for span in spans] == [(20, 25)]
+
+    def test_overlapping_and_excess_items_do_not_reuse_offsets(self):
+        """Items without a free non-overlapping occurrence are dropped."""
+        spans = LLMDetector._to_spans(["abc", "bc", "abc"], "abc")
+
+        assert spans == [{"start": 0, "end": 3, "text": "abc"}]
+
+    def test_token_output_flags_both_repeated_occurrences(self, cache_file):
+        """Token projection flags every separately localized occurrence."""
+        answer = "Paris appears and Paris repeats."
+        detector = make_detector('{"hallucination_list": ["Paris", "Paris"]}', cache_file)
+
+        tokens = detector.predict_prompt("p", answer, output_format="tokens")
+
+        assert [token["pred"] for token in tokens] == [1, 0, 0, 1, 0]