fix(rss): detect empty or HTML responses feedparser treats as non-bozo (#36) (#69)

feedparser reports bozo=False with entries=[] for an empty HTTP 200 body
or an HTML error page, making them look like a healthy empty feed (no
structured error reaches control, SCHEMA_DRIFT never fires). Fail
explicitly with a structured ParseError (maps to SCHEMA_DRIFT in
error_kinds.py) when no feed version was detected, on both collect() and
fetch() paths. Conservative: a real zero-entry feed still carries a
detected version ('rss20'/'atom10') and passes.

Co-authored-by: 1012839419a-alt <1012839419a-alt@users.noreply.github.com>

Author: 1012839419a-alt

backend/channels/rss_channel.py

@@ -43,6 +43,37 @@ def _bozo_error_type(parsed: Any) -> str:
     return "ParseError"
 
 
+def _parsed_version(parsed: Any) -> str:
+    """Detected feed format (``'rss20'``, ``'atom10'``, ...), or ``''`` when
+    feedparser recognized no feed format in the body. Works for both
+    feedparser's dict-like result and bare namespaces used in tests."""
+    getter = getattr(parsed, "get", None)
+    if callable(getter):
+        return str(getter("version") or "")
+    return str(getattr(parsed, "version", "") or "")
+
+
+def _non_feed_response(parsed: Any) -> bool:
+    """True when feedparser accepted the body but recognized no feed format.
+
+    Issue #36: an HTTP 200 with an empty body or an HTML error page parses as
+    ``bozo=False`` with ``entries=[]`` — visually identical to a legitimate
+    empty feed, so no structured error reaches control. The distinguishing
+    signal is the detected feed version: a real (even empty) RSS/Atom feed
+    parses with ``version`` set (``'rss20'``/``'atom10'``/...), while
+    empty/HTML/non-feed bodies leave it falsy (``''``/``None``). Conservative
+    by design: this only fires when zero entries were parsed, so a partial
+    parse that yielded items still succeeds. ``ParseError`` maps to
+    SCHEMA_DRIFT in backend/control/error_kinds.py, so these responses now
+    reach the control recorder instead of looking like a healthy empty feed.
+    """
+    return (
+        not getattr(parsed, "bozo", False)
+        and not getattr(parsed, "entries", None)
+        and not _parsed_version(parsed)
+    )
+
+
 @register_channel
 class RSSChannel(AbstractChannel):
     """Collect entries from RSS/Atom feeds."""
@@ -127,6 +158,16 @@ async def collect(
                 f"Failed to parse feed: {getattr(parsed, 'bozo_exception', 'unknown error')}",
                 error_type=_bozo_error_type(parsed),
             )
+        # Issue #36: feedparser treats an empty body or an HTML error page as a
+        # successful non-bozo parse with entries=[] — indistinguishable from a
+        # legitimate empty feed by bozo alone. Fail explicitly (ParseError maps
+        # to SCHEMA_DRIFT) when no feed format was recognized.
+        if _non_feed_response(parsed):
+            return ChannelResult.fail(
+                "RSS feed returned no recognized feed content "
+                "(empty body or HTML/non-feed response)",
+                error_type="ParseError",
+            )
 
         entries = parsed.entries[:max_entries]
         items = [self._entry_to_dict(entry) for entry in entries]
@@ -223,6 +264,15 @@ async def fetch(self, ctx: FetchContext) -> FetchResult:
                 f"Failed to parse feed: {getattr(parsed, 'bozo_exception', 'unknown error')}",
                 error_type=_bozo_error_type(parsed),
             )
+        # Issue #36: see collect()'s twin comment — an empty/HTML response
+        # parses as bozo=False with entries=[] and must not look like a
+        # healthy empty feed.
+        if _non_feed_response(parsed):
+            raise ChannelFetchError(
+                "RSS feed returned no recognized feed content "
+                "(empty body or HTML/non-feed response)",
+                error_type="ParseError",
+            )
         items = [self._entry_to_dict(entry) for entry in parsed.entries[:max_entries]]
 
         next_cursor = dict(cursor)

tests/unit/channels/test_rss_channel_schema_drift.py

@@ -30,6 +30,23 @@ async def get(self, *_args, **_kwargs):
         return self.response
 
 
+class _BodyClient:
+    """Collect-side client serving a fixed body (Issue #36 cases)."""
+
+    def __init__(self, text: str):
+        self.response = MagicMock(text=text)
+        self.response.raise_for_status = MagicMock()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *_args):
+        return False
+
+    async def get(self, *_args, **_kwargs):
+        return self.response
+
+
 async def _collect_with_parsed(channel, parsed):
     with (
         patch("httpx.AsyncClient", return_value=_HttpClient()),
@@ -86,3 +103,57 @@ async def test_collect_bozo_feed_with_entries_succeeds(channel):
 
     assert result.success is True
     assert len(result.items) == 1
+
+
+@pytest.mark.asyncio
+async def test_collect_html_error_page_fails_with_parse_error(channel):
+    """Issue #36: an HTML error page parses as bozo=False with entries=[] —
+    previously returned OK as a fake empty feed; now fails with a structured
+    ParseError that maps to SCHEMA_DRIFT."""
+    html = (
+        "<!DOCTYPE html><html><head><title>Error 502</title></head>"
+        "<body><h1>Bad Gateway</h1><p>nginx</p></body></html>"
+    )
+    with patch("httpx.AsyncClient", return_value=_BodyClient(html)):
+        result = await channel.collect({"feed_url": "https://example.com/rss"}, {})
+
+    assert result.success is False
+    assert result.error_type == "ParseError"
+    assert map_error_type(result.error_type) is ErrorKind.SCHEMA_DRIFT
+
+
+@pytest.mark.asyncio
+async def test_collect_empty_body_fails_with_parse_error(channel):
+    """Issue #36: an empty HTTP 200 body parses as bozo=False with entries=[]
+    — must fail as ParseError (SCHEMA_DRIFT), not look like a healthy feed."""
+    with patch("httpx.AsyncClient", return_value=_BodyClient("")):
+        result = await channel.collect({"feed_url": "https://example.com/rss"}, {})
+
+    assert result.success is False
+    assert result.error_type == "ParseError"
+    assert map_error_type(result.error_type) is ErrorKind.SCHEMA_DRIFT
+
+
+@pytest.mark.asyncio
+async def test_collect_plain_xml_not_a_feed_fails_with_parse_error(channel):
+    """Issue #36: well-formed XML that is not a feed (no recognized version)
+    must not be reported as a successful empty feed."""
+    with patch("httpx.AsyncClient", return_value=_BodyClient("<foo><bar>1</bar></foo>")):
+        result = await channel.collect({"feed_url": "https://example.com/rss"}, {})
+
+    assert result.success is False
+    assert result.error_type == "ParseError"
+    assert map_error_type(result.error_type) is ErrorKind.SCHEMA_DRIFT
+
+
+@pytest.mark.asyncio
+async def test_collect_valid_empty_feed_succeeds(channel):
+    """A legitimate zero-entry feed (feedparser detects its version) must NOT
+    be flagged as a non-feed response."""
+    empty_rss = '<rss version="2.0"><channel><title>Empty</title></channel></rss>'
+    with patch("httpx.AsyncClient", return_value=_BodyClient(empty_rss)):
+        result = await channel.collect({"feed_url": "https://example.com/rss"}, {})
+
+    assert result.success is True
+    assert result.items == []
+    assert result.metadata.get("total_entries") == 0

tests/unit/channels/test_rss_fetch.py

@@ -147,6 +147,53 @@ async def test_fetch_bozo_feed_raises_error_type_mapped_to_schema_drift():
     assert map_error_type(exc_info.value.error_type) is ErrorKind.SCHEMA_DRIFT
 
 
+@pytest.mark.asyncio
+async def test_fetch_html_response_raises_error_type_mapped_to_schema_drift():
+    """Issue #36: an HTML error page parses as bozo=False with entries=[] on
+    the fetch() path too — must raise ChannelFetchError(ParseError), mapped
+    to SCHEMA_DRIFT, instead of returning a fake empty feed."""
+    from backend.control.error_kinds import ErrorKind, map_error_type
+
+    http = _Http(_Resp(200, text="<html><body>Error 502</body></html>", headers={}))
+    ctx = FetchContext(config={"feed_url": "https://x/feed"}, params={}, cursor=None, http=http)
+
+    with pytest.raises(ChannelFetchError) as exc_info:
+        await RSSChannel().fetch(ctx)
+
+    assert exc_info.value.error_type == "ParseError"
+    assert map_error_type(exc_info.value.error_type) is ErrorKind.SCHEMA_DRIFT
+
+
+@pytest.mark.asyncio
+async def test_fetch_empty_body_raises_error_type_mapped_to_schema_drift():
+    """Issue #36: an empty HTTP 200 body on the fetch() path must raise
+    ParseError (SCHEMA_DRIFT), not return a fake empty feed."""
+    from backend.control.error_kinds import ErrorKind, map_error_type
+
+    http = _Http(_Resp(200, text="", headers={}))
+    ctx = FetchContext(config={"feed_url": "https://x/feed"}, params={}, cursor=None, http=http)
+
+    with pytest.raises(ChannelFetchError) as exc_info:
+        await RSSChannel().fetch(ctx)
+
+    assert exc_info.value.error_type == "ParseError"
+    assert map_error_type(exc_info.value.error_type) is ErrorKind.SCHEMA_DRIFT
+
+
+@pytest.mark.asyncio
+async def test_fetch_valid_empty_feed_succeeds():
+    """A legitimate zero-entry feed (version detected) must NOT be flagged as
+    a non-feed response on the fetch() path."""
+    empty_rss = '<rss version="2.0"><channel><title>Empty</title></channel></rss>'
+    http = _Http(_Resp(200, text=empty_rss, headers={}))
+    ctx = FetchContext(config={"feed_url": "https://x/feed"}, params={}, cursor=None, http=http)
+
+    result = await RSSChannel().fetch(ctx)
+
+    assert result.items == []
+    assert result.has_more is False
+
+
 @pytest.mark.asyncio
 async def test_run_channel_drives_rss_and_persists_cursor():
     from backend.pipeline.channel_runner import run_channel