feat(messages): return the full message record from every JSON reader

get_history builds its records with message_to_dict, but list_messages,
search_messages, search_global and get_message_context each assembled a
smaller record by hand. Reading the same post through them dropped the
URLs behind text links, the forward origin, views and forwards, edit and
pin state, buttons and service actions, and search results also lost the
media label and voice transcripts.

All five now use message_to_dict, passing the chat id so cached voice
transcripts show up everywhere. search_messages prefetches transcripts in
auto mode like list_messages does. message_to_dict also gains the link
preview (url, site name, title, description) and poll question and
answers, which no reader returned before.

The sender field in list_messages, search_messages and search_global is
now the plain display name, as in get_history; the username and id it
used to embed are the separate username and sender_id fields.

Author: oviron

README.md

@@ -85,11 +85,11 @@ Aliases live in `${XDG_STATE_HOME:-~/.local/state}/telegram-mcp/aliases.json` (o
 
 The engine is chosen per call via the `engine` argument, or otherwise defaults to `TELEGRAM_TRANSCRIBE_ENGINE` (`groq`, `telegram`, `openai` or `whisper`). Results are cached by `(chat_id, message_id, engine)` in a local SQLite file so repeat reads and repeat listings never re-transcribe the same message. Concurrent requests for the same uncached recording are collapsed too: the second one waits for the first and returns its transcript, so a burst of callers costs one paid call, not one per caller. Every transcript is returned with a `note` marking it as a machine transcript, not a verbatim quote — treat it as a paraphrase, not exact wording.
 
-`get_history`, `get_messages`, and `list_messages` fill in already-cached transcripts for voice messages instead of leaving the text empty, controlled by `TELEGRAM_TRANSCRIBE`:
+`get_history`, `get_messages`, `list_messages`, `search_messages`, `search_global`, and `get_message_context` fill in already-cached transcripts for voice messages instead of leaving the text empty, controlled by `TELEGRAM_TRANSCRIBE`:
 
 - `off`: transcription is disabled at runtime. The `transcribe_voice` tool stays registered and returns `{"transcribed": false, "reason": "transcription_disabled"}` instead of transcribing, and listings never show transcripts. Use `TELEGRAM_EXPOSED_TOOLS` to hide the tool itself.
 - `on-demand` (default): listings show cached transcripts but never spend an API call fetching a new one.
-- `auto`: listings also prefetch missing transcripts, bounded per call by `TELEGRAM_TRANSCRIBE_MAX_VOICES`/`TELEGRAM_TRANSCRIBE_MAX_SECONDS` (Groq isn't free, so this prefetch is budgeted rather than unbounded).
+- `auto`: `get_history`, `get_messages`, `list_messages`, and `search_messages` also prefetch missing transcripts, bounded per call by `TELEGRAM_TRANSCRIBE_MAX_VOICES`/`TELEGRAM_TRANSCRIBE_MAX_SECONDS` (Groq isn't free, so this prefetch is budgeted rather than unbounded).
 
 The cache lives in `TELEGRAM_TRANSCRIPT_CACHE_DIR` (default `data/transcripts`), written as a 700 directory / 600 file since it holds personal-chat text in plaintext — see [Docker](#docker) for why this needs its own volume mount in a container.
 - **Profile and privacy:** get your own account info, update profile fields, set or delete profile photos, inspect privacy settings, get user info/photos/status, and manage bot commands.

telegram_mcp/tools/messages.py

@@ -198,6 +198,21 @@ def message_to_dict(msg, chat_id: Optional[int] = None) -> dict:
     if media_label:
         d["media"] = media_label
 
+    page = getattr(msg, "web_preview", None)
+    if page is not None and getattr(page, "url", None):
+        d["web_preview"] = {
+            k: sanitize_user_content(v) if k != "url" else v
+            for k in ("url", "site_name", "title", "description")
+            if (v := getattr(page, k, None))
+        }
+
+    poll = getattr(msg, "poll", None)
+    if poll is not None and getattr(poll, "poll", None) is not None:
+        d["poll"] = {
+            "question": sanitize_user_content(poll.poll.question.text),
+            "answers": [sanitize_user_content(a.text.text) for a in poll.poll.answers],
+        }
+
     if not text:
         voice_info = transcription.voice_attachment_info(msg, chat_id)
         if voice_info is not None:
@@ -1132,49 +1147,7 @@ async def list_messages(
         numeric_chat_id = get_marked_id(entity)
         await transcription.prefetch_transcripts(cl, entity, numeric_chat_id, messages)
 
-        records = []
-        for msg in messages:
-            record = {
-                "id": msg.id,
-                "sender": get_sender_info(msg),
-                "date": msg.date,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            # Upstream bug: this hand-built record never called get_media_label,
-            # so a voice/photo/etc. with no caption was indistinguishable from
-            # an actually-empty message. message_to_dict (used by get_history)
-            # already gets this right.
-            media_label = get_media_label(msg)
-            if media_label:
-                record["media"] = media_label
-
-            if not getattr(msg, "message", None):
-                voice_info = transcription.voice_attachment_info(msg, numeric_chat_id)
-                if voice_info is not None:
-                    if voice_info["duration"] is not None:
-                        record["duration"] = voice_info["duration"]
-                    if voice_info["transcript_status"] == "ready":
-                        record["transcript"] = voice_info["transcript"]
-                        record["transcript_source"] = voice_info["transcript_source"]
-                        record["transcript_note"] = "Machine transcript, not a verbatim quote."
-                    elif voice_info["transcript_status"] == "pending":
-                        record["transcript_status"] = "pending"
-
-            grouped_id = getattr(msg, "grouped_id", None)
-            if grouped_id is not None:
-                record["grouped_id"] = grouped_id
-            reply_to_id = getattr(msg.reply_to, "reply_to_msg_id", None) if msg.reply_to else None
-            if reply_to_id:
-                record["reply_to"] = reply_to_id
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
-            engagement = get_engagement_dict(msg)
-            if engagement:
-                record["engagement"] = engagement
-            records.append(record)
-
+        records = [message_to_dict(msg, numeric_chat_id) for msg in messages]
         return format_tool_result(records)
     except Exception as e:
         return log_and_format_error("list_messages", e, chat_id=chat_id)
@@ -1364,52 +1337,17 @@ async def get_message_context(
         # Combine messages in chronological order
         all_messages = list(messages_before) + list(central_message) + list(messages_after)
         all_messages.sort(key=lambda m: m.id)
+        numeric_chat_id = get_marked_id(chat)
         records = []
         for msg in all_messages:
-            sender_name = get_sender_name(msg)
-            record = {
-                "id": msg.id,
-                "sender": sender_name,
-                "date": msg.date,
-                "is_target": msg.id == message_id,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            if getattr(msg, "sender_id", None):
-                record["sender_id"] = msg.sender_id
-            _username = get_sender_username(msg)
-            if _username:
-                record["username"] = _username
-            grouped_id = getattr(msg, "grouped_id", None)
-            if grouped_id is not None:
-                record["grouped_id"] = grouped_id
-            link_urls = _link_urls(msg)
-            if link_urls:
-                record["link_urls"] = link_urls
+            record = message_to_dict(msg, numeric_chat_id)
+            record["is_target"] = msg.id == message_id
 
-            # Check if this message is a reply and get the replied message
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
             if msg.reply_to and msg.reply_to.reply_to_msg_id:
-                record["reply_to"] = msg.reply_to.reply_to_msg_id
                 try:
                     replied_msg = await cl.get_messages(chat, ids=msg.reply_to.reply_to_msg_id)
                     if replied_msg:
-                        replied_record = {
-                            "sender": get_sender_name(replied_msg),
-                            "text": sanitize_user_content(replied_msg.message),
-                            **get_custom_emoji_metadata(replied_msg),
-                        }
-                        if getattr(replied_msg, "sender_id", None):
-                            replied_record["sender_id"] = replied_msg.sender_id
-                        _r_username = get_sender_username(replied_msg)
-                        if _r_username:
-                            replied_record["username"] = _r_username
-                        reply_link_urls = _link_urls(replied_msg)
-                        if reply_link_urls:
-                            replied_record["link_urls"] = reply_link_urls
-                        record["replied_message"] = replied_record
+                        record["replied_message"] = message_to_dict(replied_msg, numeric_chat_id)
                 except Exception:
                     record["replied_message"] = None
 
@@ -2023,21 +1961,9 @@ async def search_messages(
         entity = await resolve_entity(chat_id, cl)
         messages = await cl.get_messages(entity, limit=limit, search=query)
 
-        records = []
-        for msg in messages:
-            record = {
-                "id": msg.id,
-                "sender": get_sender_info(msg),
-                "date": msg.date,
-                "text": sanitize_user_content(msg.message),
-                **get_custom_emoji_metadata(msg),
-            }
-            if msg.reply_to and msg.reply_to.reply_to_msg_id:
-                record["reply_to"] = msg.reply_to.reply_to_msg_id
-            reply_quote = get_reply_quote(msg)
-            if reply_quote:
-                record["reply_quote"] = reply_quote
-            records.append(record)
+        numeric_chat_id = get_marked_id(entity)
+        await transcription.prefetch_transcripts(cl, entity, numeric_chat_id, messages)
+        records = [message_to_dict(msg, numeric_chat_id) for msg in messages]
         return format_tool_result(records)
     except Exception as e:
         return log_and_format_error(
@@ -2085,11 +2011,7 @@ async def search_global(
                 {
                     "chat_name": sanitize_name(chat_name),
                     "chat_id": msg.chat_id,
-                    "id": msg.id,
-                    "sender": get_sender_info(msg),
-                    "date": msg.date,
-                    "text": sanitize_user_content(msg.message),
-                    **get_custom_emoji_metadata(msg),
+                    **message_to_dict(msg, msg.chat_id),
                 }
             )
 

tests/test_full_message_records.py

@@ -0,0 +1,179 @@
+"""Every JSON reader returns the same complete record as get_history.
+
+list_messages, search_messages, search_global and get_message_context used to
+hand-build a text-only record, so a channel post read through them lost the
+link hidden behind its text, its forward origin and its view count.
+"""
+
+import json
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from telethon.tl import types
+
+from telegram_mcp import transcription
+from telegram_mcp.tools import messages
+
+DATE = datetime(2026, 9, 16, tzinfo=timezone.utc)
+HIDDEN_URL = "https://example.com/model"
+CHANNEL_ID = 1234567890
+MARKED_CHANNEL_ID = -1001234567890
+
+
+def _post(**kwargs):
+    msg = types.Message(
+        id=1,
+        peer_id=types.PeerChannel(CHANNEL_ID),
+        date=DATE,
+        message="Model is here",
+        entities=[types.MessageEntityTextUrl(9, 4, HIDDEN_URL)],
+        fwd_from=types.MessageFwdHeader(
+            date=DATE, from_id=types.PeerChannel(987654321), channel_post=17
+        ),
+        views=1500,
+        forwards=12,
+        **kwargs,
+    )
+    msg._chat = SimpleNamespace(id=CHANNEL_ID, title="Example channel")
+    return msg
+
+
+def _voice():
+    doc = types.Document(
+        id=1,
+        access_hash=0,
+        file_reference=b"",
+        date=DATE,
+        mime_type="audio/ogg",
+        size=1,
+        dc_id=2,
+        attributes=[types.DocumentAttributeAudio(duration=13, voice=True)],
+    )
+    msg = types.Message(
+        id=1,
+        peer_id=types.PeerChannel(CHANNEL_ID),
+        date=DATE,
+        message="",
+        media=types.MessageMediaDocument(document=doc),
+    )
+    msg._chat = SimpleNamespace(id=CHANNEL_ID, title="Example channel")
+    return msg
+
+
+@pytest.fixture
+def client(monkeypatch):
+    cl = AsyncMock()
+    monkeypatch.setattr(messages, "get_client", lambda account=None: cl)
+    monkeypatch.setattr(
+        messages,
+        "resolve_entity",
+        AsyncMock(
+            return_value=types.Channel(
+                id=CHANNEL_ID, title="Example channel", photo=types.ChatPhotoEmpty(), date=DATE
+            )
+        ),
+    )
+    monkeypatch.setattr(messages, "ensure_connected", AsyncMock())
+    monkeypatch.setattr(messages.transcription, "prefetch_transcripts", AsyncMock())
+    return cl
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "tool, arguments",
+    [
+        (messages.get_history, {"chat_id": MARKED_CHANNEL_ID}),
+        (messages.list_messages, {"chat_id": MARKED_CHANNEL_ID}),
+        (messages.search_messages, {"chat_id": MARKED_CHANNEL_ID, "query": "Model"}),
+        (messages.search_global, {"query": "Model"}),
+        (messages.get_message_context, {"chat_id": MARKED_CHANNEL_ID, "message_id": 1}),
+    ],
+)
+async def test_json_readers_return_the_complete_record(client, tool, arguments):
+    msg = _post()
+    client.get_messages.return_value = [msg]
+    if tool is messages.get_message_context:
+        client.get_messages.side_effect = [[], msg, []]
+
+    record = json.loads(await tool(**arguments, account="test"))["results"][0]
+
+    assert record["text"] == "Model is here"
+    assert record["link_urls"] == [HIDDEN_URL]
+    assert record["forwarded"]["channel_post"] == 17
+    assert record["engagement"]["views"] == 1500
+
+
+@pytest.mark.asyncio
+async def test_search_global_keeps_chat_attribution(client):
+    client.get_messages.return_value = [_post()]
+
+    record = json.loads(await messages.search_global("Model", account="test"))["results"][0]
+
+    assert record["chat_name"] == "Example channel"
+    assert record["chat_id"] == MARKED_CHANNEL_ID
+
+
+@pytest.mark.asyncio
+async def test_context_keeps_target_marker(client):
+    msg = _post()
+    client.get_messages.side_effect = [[], msg, []]
+
+    result = json.loads(await messages.get_message_context(MARKED_CHANNEL_ID, 1, account="test"))
+
+    assert result["results"][0]["is_target"] is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("tool", ["search_global", "get_message_context"])
+async def test_cached_voice_transcript_is_part_of_the_record(client, transcript_cache_dir, tool):
+    transcription.save_transcript(MARKED_CHANNEL_ID, 1, "groq", "Hello there", duration=13)
+    voice = _voice()
+    if tool == "search_global":
+        client.get_messages.return_value = [voice]
+        result = await messages.search_global("Hello", account="test")
+    else:
+        client.get_messages.side_effect = [[], voice, []]
+        result = await messages.get_message_context(MARKED_CHANNEL_ID, 1, account="test")
+
+    record = json.loads(result)["results"][0]
+
+    assert record["media"] == "voice"
+    assert record["transcript"] == "Hello there"
+
+
+def test_link_preview_is_part_of_the_record():
+    page = types.WebPage(
+        id=1,
+        url=HIDDEN_URL,
+        display_url="example.com/model",
+        hash=0,
+        site_name="Example",
+        title="Example model",
+    )
+    msg = _post(media=types.MessageMediaWebPage(webpage=page))
+
+    assert messages.message_to_dict(msg)["web_preview"] == {
+        "url": HIDDEN_URL,
+        "site_name": "Example",
+        "title": "Example model",
+    }
+
+
+def test_poll_question_and_answers_are_part_of_the_record():
+    poll = types.Poll(
+        id=1,
+        question=types.TextWithEntities(text="Which model?", entities=[]),
+        answers=[
+            types.PollAnswer(text=types.TextWithEntities(text="Qwen", entities=[]), option=b"0"),
+            types.PollAnswer(text=types.TextWithEntities(text="Gemma", entities=[]), option=b"1"),
+        ],
+        hash=0,
+    )
+    msg = _post(media=types.MessageMediaPoll(poll=poll, results=types.PollResults()))
+
+    assert messages.message_to_dict(msg)["poll"] == {
+        "question": "Which model?",
+        "answers": ["Qwen", "Gemma"],
+    }