fix: harden identity query retrieval

Author: therayyanawaz

ai_filter.py

@@ -5687,6 +5687,8 @@ def _valid(definition: str, *, reverse: bool = False) -> str:
     forward_patterns = (
         rf"\b{escaped}\s*:\s*(?P<definition>[^.;:]+)",
         rf"\b{escaped}\s+(?:is|was|are|were)\s+(?P<definition>[^.;:]+)",
+        rf"\b{escaped},\s+(?:which|who)\s+(?:is|was|are|were)\s+(?P<definition>[^.;:]+)",
+        rf"\b{escaped},\s+(?:the|a|an)\s+(?P<definition>[^,.;:]+)",
         rf"\b{escaped},\s+(?P<definition>[^.;:]+)",
     )
     for pattern in forward_patterns:
@@ -5739,11 +5741,15 @@ def _render_local_nlp_query_answer(
     if lead:
         parts.append(sanitize_telegram_html(lead))
     seen = {_digest_line_key(lead)}
+    lead_plain = normalize_space(strip_telegram_html(lead)).lower()
     max_bullets = 4 if detailed else 2
     for sentence in sentences[1:]:
         key = _digest_line_key(sentence)
         if not key or key in seen:
             continue
+        sentence_plain = normalize_space(strip_telegram_html(sentence)).lower()
+        if lead_plain and SequenceMatcher(None, lead_plain, sentence_plain).ratio() > 0.85:
+            continue
         seen.add(key)
         parts.append(f"• {sanitize_telegram_html(sentence)}")
         if len(parts) >= 2 + max_bullets:
@@ -7396,6 +7402,7 @@ async def generate_answer_from_context_result(
         output_language=output_language,
         detailed=detailed,
         strategic_trend=strategic_trend_query,
+        identity_query=_query_is_identity_question(question),
     )
     retry_issue = ""
 

main.py

@@ -174,6 +174,7 @@
     extract_query_numbers,
     sanitize_telegram_html,
     search_entity_web_summary,
+    query_entity_title,
     search_recent_news_web,
     search_recent_messages,
     query_prefers_direct_answer,
@@ -1734,6 +1735,10 @@ def _query_window_strategy(*, requested_hours: int, explicit_time_filter: bool)
     return min(primary_hours, 24), max(primary_hours, _query_expanded_window_hours())
 
 
+def _query_identity_window_hours() -> int:
+    return 24 * 14
+
+
 def _query_needs_expanded_window(
     query: str,
     results: Sequence[Dict[str, object]],
@@ -9805,10 +9810,15 @@ async def _handle_query_request(
 
     plan = build_query_plan(text, default_hours=_query_default_hours_back())
     parsed_hours = plan.hours_back
+    explicit_time_filter = bool(getattr(plan, "explicit_time_filter", False))
     primary_hours, expanded_hours = _query_window_strategy(
         requested_hours=parsed_hours,
-        explicit_time_filter=bool(getattr(plan, "explicit_time_filter", False)),
+        explicit_time_filter=explicit_time_filter,
     )
+    identity_query = bool(query_entity_title(plan.original_query or text))
+    if identity_query and not explicit_time_filter:
+        primary_hours = max(primary_hours, _query_identity_window_hours())
+        expanded_hours = max(expanded_hours, primary_hours)
 
     progress = await _safe_reply_markdown(
         event_ref,  # type: ignore[arg-type]
@@ -9844,6 +9854,16 @@ async def _handle_query_request(
             force=True,
         )
 
+        entity_web_results: list[dict[str, object]] = []
+        entity_web_summary_used = False
+        should_run_web_search = bool(_is_query_web_crosscheck_required())
+        if should_run_web_search and identity_query:
+            entity_web_results = await _search_entity_web_summary_bounded(
+                effective_query,
+                logger=LOGGER,
+            )
+            entity_web_summary_used = bool(entity_web_results)
+
         queue_results: list[dict[str, object]] = []
         archive_results: list[dict[str, object]] = []
 
@@ -9927,9 +9947,7 @@ async def _handle_query_request(
         web_results: list[dict[str, object]] = []
         web_fallback_used = False
         ran_web_search = False
-        entity_web_summary_used = False
-
-        should_run_web_search = bool(_is_query_web_crosscheck_required())
+        web_results = list(entity_web_results)
 
         if should_run_web_search:
             ran_web_search = True
@@ -9951,17 +9969,11 @@ async def _handle_query_request(
                 progress_cb=progress_tracker.handle_search_progress,
                 logger=LOGGER,
             )
-            entity_rows = await _search_entity_web_summary_bounded(
-                effective_query,
-                logger=LOGGER,
+            web_results = _merge_query_context(
+                entity_web_results,
+                web_results,
+                limit=max(_query_web_max_results(), 3),
             )
-            if entity_rows:
-                entity_web_summary_used = True
-                web_results = _merge_query_context(
-                    entity_rows,
-                    web_results,
-                    limit=max(_query_web_max_results(), 3),
-                )
 
         if should_run_web_search and web_results and not entity_web_summary_used:
             required_sources = _query_web_require_min_sources()
@@ -9981,8 +9993,8 @@ async def _handle_query_request(
         web_fallback_used = bool(web_results)
 
         results = _merge_query_context(
-            pre_web_results,
             web_results,
+            pre_web_results,
             limit=max(context_limit, _query_max_messages()),
         )
 

prompts.py

@@ -220,6 +220,7 @@ def build_query_system_prompt(
     output_language: str,
     detailed: bool,
     strategic_trend: bool = False,
+    identity_query: bool = False,
 ) -> str:
     mode_line = (
         "Detail mode enabled: still lead with a sharp direct answer, then use no more than 4 short support bullets."
@@ -231,12 +232,18 @@ def build_query_system_prompt(
         if strategic_trend
         else ""
     )
+    identity_line = (
+        "Identity query: use an Anchor + Delta structure. Line 1 must define the entity as 'X is [entity type] from/connected to [place/org]'. Line 2 must say why it is in the news now. Bullets may add numbers or official claims. Never define X from a report lead like 'initial reports indicate'."
+        if identity_query
+        else ""
+    )
     base = QUERY_SYSTEM_PROMPT.replace("NO_MATCH_SENTINEL", QUERY_NO_MATCH_TEXT)
     return (
         f"{base}\n\n"
         f"Output language must be {output_language}. Translate source text when needed.\n"
         f"{HUMAN_NEWSROOM_VOICE}\n"
         f"{HTML_RULES}\n"
         f"{mode_line}\n"
-        f"{strategic_line}".rstrip()
+        f"{strategic_line}\n"
+        f"{identity_line}".rstrip()
     )

tests/test_feed_quality.py

@@ -902,6 +902,31 @@ def test_local_query_answer_does_not_define_entity_from_report_lead():
     assert "€1.7 billion" in plain
 
 
+def test_local_query_answer_handles_appositive_identity_and_dedupes_support():
+    answer = ai_filter._local_nlp_query_answer(
+        "what is Lufthansa?",
+        [
+            {
+                "text": "Lufthansa, the German flag carrier airline, announced losses after fuel prices rose.",
+                "source": "Web:Article",
+                "timestamp": 1778150000,
+                "is_web": True,
+            },
+            {
+                "text": "Lufthansa, the German flag carrier airline, announced losses after fuel prices rose.",
+                "source": "Web:Article",
+                "timestamp": 1778150000,
+                "is_web": True,
+            },
+        ],
+        detailed=False,
+    )
+    plain = ai_filter.strip_telegram_html(answer)
+
+    assert "Lufthansa is German flag carrier airline" in plain
+    assert plain.count("German flag carrier airline") == 1
+
+
 def test_local_query_answer_defines_entity_from_wikipedia_style_context():
     answer = ai_filter._local_nlp_query_answer(
         "what or who is Lufthansa?",
@@ -2024,11 +2049,13 @@ async def fake_generate_answer_from_context(*, query, context_messages, auth_man
 async def test_handle_query_request_adds_entity_web_summary_for_identity_query(monkeypatch):
     main.query_last_request_ts.clear()
     answer_contexts: list[list[dict[str, object]]] = []
+    search_hours: list[int] = []
 
     async def fake_safe_reply_markdown(_event, text, *, edit_message=None, reply_to=None, prefer_bot_identity=False, bot_chat_id=None):
         return edit_message or {"message_id": 711}
 
     async def fake_search_recent_messages(*_args, **_kwargs):
+        search_hours.append(int(_kwargs.get("default_hours_back") or 0))
         return [
             {
                 "text": "Initial reports indicate Lufthansa has lost €1.7 billion due to rising jet fuel prices.",
@@ -2079,6 +2106,8 @@ async def fake_generate_answer_from_context(*, query, context_messages, auth_man
     )
 
     assert answer_contexts
+    assert search_hours == [24 * 14]
+    assert answer_contexts[0][0]["source"] == "Web:Wikipedia"
     assert any(row["source"] == "Web:Wikipedia" for row in answer_contexts[0])