Merge pull request chigwell#254 from oviron/feat/full-message-records

Return the full message record from list, search and context tools

Author: chigwell

README.md

@@ -85,11 +85,11 @@ Aliases live in `${XDG_STATE_HOME:-~/.local/state}/telegram-mcp/aliases.json` (o
 
 The engine is chosen per call via the `engine` argument, or otherwise defaults to `TELEGRAM_TRANSCRIBE_ENGINE` (`groq`, `telegram`, `openai` or `whisper`). Results are cached by `(chat_id, message_id, engine)` in a local SQLite file so repeat reads and repeat listings never re-transcribe the same message. Concurrent requests for the same uncached recording are collapsed too: the second one waits for the first and returns its transcript, so a burst of callers costs one paid call, not one per caller. Every transcript is returned with a `note` marking it as a machine transcript, not a verbatim quote — treat it as a paraphrase, not exact wording.
 
-`get_history`, `get_messages`, and `list_messages` fill in already-cached transcripts for voice messages instead of leaving the text empty, controlled by `TELEGRAM_TRANSCRIBE`:
+`get_history`, `get_messages`, `list_messages`, `search_messages`, `search_global`, and `get_message_context` fill in already-cached transcripts for voice messages instead of leaving the text empty, controlled by `TELEGRAM_TRANSCRIBE`:
 
 - `off`: transcription is disabled at runtime. The `transcribe_voice` tool stays registered and returns `{"transcribed": false, "reason": "transcription_disabled"}` instead of transcribing, and listings never show transcripts. Use `TELEGRAM_EXPOSED_TOOLS` to hide the tool itself.
 - `on-demand` (default): listings show cached transcripts but never spend an API call fetching a new one.
-- `auto`: listings also prefetch missing transcripts, bounded per call by `TELEGRAM_TRANSCRIBE_MAX_VOICES`/`TELEGRAM_TRANSCRIBE_MAX_SECONDS` (Groq isn't free, so this prefetch is budgeted rather than unbounded).
+- `auto`: `get_history`, `get_messages`, `list_messages`, and `search_messages` also prefetch missing transcripts, bounded per call by `TELEGRAM_TRANSCRIBE_MAX_VOICES`/`TELEGRAM_TRANSCRIBE_MAX_SECONDS` (Groq isn't free, so this prefetch is budgeted rather than unbounded).
 
 The cache lives in `TELEGRAM_TRANSCRIPT_CACHE_DIR` (default `data/transcripts`), written as a 700 directory / 600 file since it holds personal-chat text in plaintext — see [Docker](#docker) for why this needs its own volume mount in a container.
 - **Profile and privacy:** get your own account info, update profile fields, set or delete profile photos, inspect privacy settings, get user info/photos/status, and manage bot commands.

telegram_mcp/tools/messages.py

@@ -198,6 +198,21 @@ def message_to_dict(msg, chat_id: Optional[int] = None) -> dict:
     if media_label:
         d["media"] = media_label
 
+    page = getattr(msg, "web_preview", None)
+    if page is not None and getattr(page, "url", None):
+        d["web_preview"] = {
+            k: sanitize_user_content(v) if k != "url" else v
+            for k in ("url", "site_name", "title", "description")
+            if (v := getattr(page, k, None))
+        }
+
+    poll = getattr(msg, "poll", None)
+    if poll is not None and getattr(poll, "poll", None) is not None:
+        d["poll"] = {
+            "question": sanitize_user_content(poll.poll.question.text),
+            "answers": [sanitize_user_content(a.text.text) for a in poll.poll.answers],
+        }
+
     if not text:
         voice_info = transcription.voice_attachment_info(msg, chat_id)
         if voice_info is not None:
@@ -1132,49 +1147,7 @@ async def list_messages(
         numeric_chat_id = get_marked_id(entity)
         await transcription.prefetch_transcripts(cl, entity, numeric_chat_id, messages)
 
-        records = []
-        for msg in messages:
-            record = {
-                "id": msg.id,
-                "sender": get_sender_info(msg),
-                "date": msg.date,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            # Upstream bug: this hand-built record never called get_media_label,
-            # so a voice/photo/etc. with no caption was indistinguishable from
-            # an actually-empty message. message_to_dict (used by get_history)
-            # already gets this right.
-            media_label = get_media_label(msg)
-            if media_label:
-                record["media"] = media_label
-
-            if not getattr(msg, "message", None):
-                voice_info = transcription.voice_attachment_info(msg, numeric_chat_id)
-                if voice_info is not None:
-                    if voice_info["duration"] is not None:
-                        record["duration"] = voice_info["duration"]
-                    if voice_info["transcript_status"] == "ready":
-                        record["transcript"] = voice_info["transcript"]
-                        record["transcript_source"] = voice_info["transcript_source"]
-                        record["transcript_note"] = "Machine transcript, not a verbatim quote."
-                    elif voice_info["transcript_status"] == "pending":
-                        record["transcript_status"] = "pending"
-
-            grouped_id = getattr(msg, "grouped_id", None)
-            if grouped_id is not None:
-                record["grouped_id"] = grouped_id
-            reply_to_id = getattr(msg.reply_to, "reply_to_msg_id", None) if msg.reply_to else None
-            if reply_to_id:
-                record["reply_to"] = reply_to_id
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
-            engagement = get_engagement_dict(msg)
-            if engagement:
-                record["engagement"] = engagement
-            records.append(record)
-
+        records = [message_to_dict(msg, numeric_chat_id) for msg in messages]
         return format_tool_result(records)
     except Exception as e:
         return log_and_format_error("list_messages", e, chat_id=chat_id)
@@ -1364,52 +1337,17 @@ async def get_message_context(
         # Combine messages in chronological order
         all_messages = list(messages_before) + list(central_message) + list(messages_after)
         all_messages.sort(key=lambda m: m.id)
+        numeric_chat_id = get_marked_id(chat)
         records = []
         for msg in all_messages:
-            sender_name = get_sender_name(msg)
-            record = {
-                "id": msg.id,
-                "sender": sender_name,
-                "date": msg.date,
-                "is_target": msg.id == message_id,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            if getattr(msg, "sender_id", None):
-                record["sender_id"] = msg.sender_id
-            _username = get_sender_username(msg)
-            if _username:
-                record["username"] = _username
-            grouped_id = getattr(msg, "grouped_id", None)
-            if grouped_id is not None:
-                record["grouped_id"] = grouped_id
-            link_urls = _link_urls(msg)
-            if link_urls:
-                record["link_urls"] = link_urls
+            record = message_to_dict(msg, numeric_chat_id)
+            record["is_target"] = msg.id == message_id
 
-            # Check if this message is a reply and get the replied message
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
             if msg.reply_to and msg.reply_to.reply_to_msg_id:
-                record["reply_to"] = msg.reply_to.reply_to_msg_id
                 try:
                     replied_msg = await cl.get_messages(chat, ids=msg.reply_to.reply_to_msg_id)
                     if replied_msg:
-                        replied_record = {
-                            "sender": get_sender_name(replied_msg),
-                            "text": sanitize_user_content(replied_msg.message),
-                            **get_custom_emoji_metadata(replied_msg),
-                        }
-                        if getattr(replied_msg, "sender_id", None):
-                            replied_record["sender_id"] = replied_msg.sender_id
-                        _r_username = get_sender_username(replied_msg)
-                        if _r_username:
-                            replied_record["username"] = _r_username
-                        reply_link_urls = _link_urls(replied_msg)
-                        if reply_link_urls:
-                            replied_record["link_urls"] = reply_link_urls
-                        record["replied_message"] = replied_record
+                        record["replied_message"] = message_to_dict(replied_msg, numeric_chat_id)
                 except Exception:
                     record["replied_message"] = None
 
@@ -2023,21 +1961,9 @@ async def search_messages(
         entity = await resolve_entity(chat_id, cl)
         messages = await cl.get_messages(entity, limit=limit, search=query)
 
-        records = []
-        for msg in messages:
-            record = {
-                "id": msg.id,
-                "sender": get_sender_info(msg),
-                "date": msg.date,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            if msg.reply_to and msg.reply_to.reply_to_msg_id:
-                record["reply_to"] = msg.reply_to.reply_to_msg_id
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
-            records.append(record)
+        numeric_chat_id = get_marked_id(entity)
+        await transcription.prefetch_transcripts(cl, entity, numeric_chat_id, messages)
+        records = [message_to_dict(msg, numeric_chat_id) for msg in messages]
         return format_tool_result(records)
     except Exception as e:
         return log_and_format_error(
@@ -2085,11 +2011,7 @@ async def search_global(
                 {
                     "chat_name": sanitize_name(chat_name),
                     "chat_id": msg.chat_id,
-                    "id": msg.id,
-                    "sender": get_sender_info(msg),
-                    "date": msg.date,
-                    "text": sanitize_user_content(msg.message),
-                    **get_custom_emoji_metadata(msg),
+                    **message_to_dict(msg, msg.chat_id),
                 }
             )
 

tests/test_full_message_records.py

@@ -0,0 +1,179 @@
+"""Every JSON reader returns the same complete record as get_history.
+
+list_messages, search_messages, search_global and get_message_context used to
+hand-build a text-only record, so a channel post read through them lost the
+link hidden behind its text, its forward origin and its view count.
+"""
+
+import json
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from telethon.tl import types
+
+from telegram_mcp import transcription
+from telegram_mcp.tools import messages
+
+DATE = datetime(2026, 9, 16, tzinfo=timezone.utc)
+HIDDEN_URL = "https://example.com/model"
+CHANNEL_ID = 1234567890
+MARKED_CHANNEL_ID = -1001234567890
+
+
+def _post(**kwargs):
+    msg = types.Message(
+        id=1,
+        peer_id=types.PeerChannel(CHANNEL_ID),
+        date=DATE,
+        message="Model is here",
+        entities=[types.MessageEntityTextUrl(9, 4, HIDDEN_URL)],
+        fwd_from=types.MessageFwdHeader(
+            date=DATE, from_id=types.PeerChannel(987654321), channel_post=17
+        ),
+        views=1500,
+        forwards=12,
+        **kwargs,
+    )
+    msg._chat = SimpleNamespace(id=CHANNEL_ID, title="Example channel")
+    return msg
+
+
+def _voice():
+    doc = types.Document(
+        id=1,
+        access_hash=0,
+        file_reference=b"",
+        date=DATE,
+        mime_type="audio/ogg",
+        size=1,
+        dc_id=2,
+        attributes=[types.DocumentAttributeAudio(duration=13, voice=True)],
+    )
+    msg = types.Message(
+        id=1,
+        peer_id=types.PeerChannel(CHANNEL_ID),
+        date=DATE,
+        message="",
+        media=types.MessageMediaDocument(document=doc),
+    )
+    msg._chat = SimpleNamespace(id=CHANNEL_ID, title="Example channel")
+    return msg
+
+
+@pytest.fixture
+def client(monkeypatch):
+    cl = AsyncMock()
+    monkeypatch.setattr(messages, "get_client", lambda account=None: cl)
+    monkeypatch.setattr(
+        messages,
+        "resolve_entity",
+        AsyncMock(
+            return_value=types.Channel(
+                id=CHANNEL_ID, title="Example channel", photo=types.ChatPhotoEmpty(), date=DATE
+            )
+        ),
+    )
+    monkeypatch.setattr(messages, "ensure_connected", AsyncMock())
+    monkeypatch.setattr(messages.transcription, "prefetch_transcripts", AsyncMock())
+    return cl
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "tool, arguments",
+    [
+        (messages.get_history, {"chat_id": MARKED_CHANNEL_ID}),
+        (messages.list_messages, {"chat_id": MARKED_CHANNEL_ID}),
+        (messages.search_messages, {"chat_id": MARKED_CHANNEL_ID, "query": "Model"}),
+        (messages.search_global, {"query": "Model"}),
+        (messages.get_message_context, {"chat_id": MARKED_CHANNEL_ID, "message_id": 1}),
+    ],
+)
+async def test_json_readers_return_the_complete_record(client, tool, arguments):
+    msg = _post()
+    client.get_messages.return_value = [msg]
+    if tool is messages.get_message_context:
+        client.get_messages.side_effect = [[], msg, []]
+
+    record = json.loads(await tool(**arguments, account="test"))["results"][0]
+
+    assert record["text"] == "Model is here"
+    assert record["link_urls"] == [HIDDEN_URL]
+    assert record["forwarded"]["channel_post"] == 17
+    assert record["engagement"]["views"] == 1500
+
+
+@pytest.mark.asyncio
+async def test_search_global_keeps_chat_attribution(client):
+    client.get_messages.return_value = [_post()]
+
+    record = json.loads(await messages.search_global("Model", account="test"))["results"][0]
+
+    assert record["chat_name"] == "Example channel"
+    assert record["chat_id"] == MARKED_CHANNEL_ID
+
+
+@pytest.mark.asyncio
+async def test_context_keeps_target_marker(client):
+    msg = _post()
+    client.get_messages.side_effect = [[], msg, []]
+
+    result = json.loads(await messages.get_message_context(MARKED_CHANNEL_ID, 1, account="test"))
+
+    assert result["results"][0]["is_target"] is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("tool", ["search_global", "get_message_context"])
+async def test_cached_voice_transcript_is_part_of_the_record(client, transcript_cache_dir, tool):
+    transcription.save_transcript(MARKED_CHANNEL_ID, 1, "groq", "Hello there", duration=13)
+    voice = _voice()
+    if tool == "search_global":
+        client.get_messages.return_value = [voice]
+        result = await messages.search_global("Hello", account="test")
+    else:
+        client.get_messages.side_effect = [[], voice, []]
+        result = await messages.get_message_context(MARKED_CHANNEL_ID, 1, account="test")
+
+    record = json.loads(result)["results"][0]
+
+    assert record["media"] == "voice"
+    assert record["transcript"] == "Hello there"
+
+
+def test_link_preview_is_part_of_the_record():
+    page = types.WebPage(
+        id=1,
+        url=HIDDEN_URL,
+        display_url="example.com/model",
+        hash=0,
+        site_name="Example",
+        title="Example model",
+    )
+    msg = _post(media=types.MessageMediaWebPage(webpage=page))
+
+    assert messages.message_to_dict(msg)["web_preview"] == {
+        "url": HIDDEN_URL,
+        "site_name": "Example",
+        "title": "Example model",
+    }
+
+
+def test_poll_question_and_answers_are_part_of_the_record():
+    poll = types.Poll(
+        id=1,
+        question=types.TextWithEntities(text="Which model?", entities=[]),
+        answers=[
+            types.PollAnswer(text=types.TextWithEntities(text="Qwen", entities=[]), option=b"0"),
+            types.PollAnswer(text=types.TextWithEntities(text="Gemma", entities=[]), option=b"1"),
+        ],
+        hash=0,
+    )
+    msg = _post(media=types.MessageMediaPoll(poll=poll, results=types.PollResults()))
+
+    assert messages.message_to_dict(msg)["poll"] == {
+        "question": "Which model?",
+        "answers": ["Qwen", "Gemma"],
+    }