Merge pull request #33 from pullboxapp/feature/annas-search-redirect

fix: make Anna's Archive search fallback provider-safe

Author: DeusExTaco

README.md

@@ -181,6 +181,22 @@ quota, source availability, and malformed responses remain distinct failures.
 Opening search-result details must not call resolve because a fast-link request
 may consume source quota.
 
+Search attempts the selected official Anna's Archive page first. When that page
+is blocked by a browser challenge, is temporarily unavailable, or returns no
+candidates, the provider performs a bounded fallback against the LibGen comics
+catalog. Only candidates with a matching lowercase LibGen ID and MD5 content
+fingerprint are considered for Anna's Archive discovery, with canonical files
+listed before mobile derivatives. Catalog presence does not guarantee that Anna
+offers a member fast-download route, so availability is verified only when the
+user grabs the result. Catalog-derived candidates intentionally do not expose a
+cross-provider fingerprint to Pullbox: if Anna cannot resolve the record,
+Pullbox reports that failure instead of silently downloading it from LibGen.
+Resolution still uses the official member fast-download JSON API, and the member
+secret is never sent to LibGen. This fallback covers only Anna's Archive records
+sourced from LibGen and does not claim parity with Anna's Archive's complete
+catalog. The Anna's Archive image includes the catalog-discovery dependency and
+does not require a separate LibGen provider container.
+
 Successful resolves may report provider-generic remaining/limit/window quota
 telemetry. The response intentionally excludes account identity and download
 history. Pullbox stores only the latest capacity observation, applies its

docker/Dockerfile.annas-archive

@@ -17,9 +17,11 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \
 WORKDIR /build
 
 COPY packages/provider_contract packages/provider_contract
+COPY providers/libgen providers/libgen
 COPY providers/annas_archive providers/annas_archive
 
 RUN python -m pip wheel --wheel-dir /wheels ./packages/provider_contract && \
+    python -m pip wheel --find-links /wheels --wheel-dir /wheels ./providers/libgen && \
     python -m pip wheel --find-links /wheels --wheel-dir /wheels ./providers/annas_archive
 
 FROM security-patched AS runtime

packages/provider_contract/src/pullbox_provider_contract/source_http.py

@@ -5,7 +5,7 @@
 import asyncio
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from typing import TYPE_CHECKING
-from urllib.parse import urlsplit
+from urllib.parse import urljoin, urlsplit
 
 import httpx
 
@@ -24,6 +24,7 @@
 _MAX_SOURCE_BYTES = 8 * 1024 * 1024
 _MAX_REDIRECT_URL_LENGTH = 4_000
 _REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+_MAX_SOURCE_REDIRECTS = 1
 
 BrowserResolver = Callable[..., Awaitable[ProviderResolverOutcome | None]]
 
@@ -62,19 +63,37 @@ async def fetch_source_html(
     response: httpx.Response | None = None
     try:
         try:
-            response = await client.send(
-                client.build_request("GET", safe_url, headers={"Accept": "text/html"}),
-                stream=True,
-                follow_redirects=False,
-            )
-            body = await _read_bounded(response)
+            for redirect_count in range(_MAX_SOURCE_REDIRECTS + 1):
+                response = await client.send(
+                    client.build_request("GET", safe_url, headers={"Accept": "text/html"}),
+                    stream=True,
+                    follow_redirects=False,
+                )
+                if response.status_code not in _REDIRECT_STATUS_CODES:
+                    body = await _read_bounded(response)
+                    break
+                if redirect_count >= _MAX_SOURCE_REDIRECTS:
+                    raise RuntimeError("Source redirect limit was exceeded.")
+                redirected_url = _same_origin_source_redirect(
+                    safe_url,
+                    response.headers.get("location"),
+                )
+                await response.aclose()
+                response = None
+                safe_url = await _validate_source_url(
+                    redirected_url,
+                    declared_domains,
+                    resolver=target_resolver,
+                )
+            else:  # pragma: no cover - bounded loop always returns or raises
+                raise RuntimeError("Source redirect limit was exceeded.")
         except asyncio.CancelledError:
             raise
         except (httpx.HTTPError, TimeoutError) as exc:
             raise RuntimeError("Source request is temporarily unavailable.") from exc
 
-        if response.status_code in {301, 302, 303, 307, 308}:
-            raise RuntimeError("Source redirect was rejected.")
+        if response is None:  # pragma: no cover - defensive invariant
+            raise RuntimeError("Source request did not return a response.")
         ordinary = OrdinaryHttpResponse(
             status_code=response.status_code,
             headers=_safe_headers(response.headers),
@@ -109,6 +128,32 @@ async def fetch_source_html(
             await client.aclose()
 
 
+def _same_origin_source_redirect(source_url: str, raw_location: str | None) -> str:
+    if not raw_location or len(raw_location) > _MAX_REDIRECT_URL_LENGTH:
+        raise RuntimeError("Source redirect destination is invalid.")
+    try:
+        source = urlsplit(source_url)
+        destination_url = urljoin(source_url, raw_location.strip())
+        destination = urlsplit(destination_url)
+        source_port = source.port or 443
+        destination_port = destination.port or 443
+    except ValueError as exc:
+        raise RuntimeError("Source redirect destination is invalid.") from exc
+    if (
+        source.scheme != "https"
+        or destination.scheme != "https"
+        or not source.hostname
+        or not destination.hostname
+        or source.hostname.casefold().rstrip(".") != destination.hostname.casefold().rstrip(".")
+        or source_port != destination_port
+        or destination.username
+        or destination.password
+        or destination.fragment
+    ):
+        raise RuntimeError("Source redirect destination was rejected.")
+    return destination_url
+
+
 async def resolve_source_redirect(
     raw_url: str,
     *,

providers/annas_archive/pyproject.toml

@@ -10,6 +10,7 @@ requires-python = ">=3.12"
 license = "GPL-3.0-or-later"
 dependencies = [
     "pullbox-direct-provider-contract==1.0.0",
+    "pullbox-provider-libgen==0.1.0",
     "uvicorn>=0.34,<1",
 ]
 

providers/annas_archive/src/pullbox_provider_annas_archive/app.py

@@ -30,6 +30,7 @@
 from pullbox_provider_contract.source_http import BrowserChallengeRequiredError
 
 from pullbox_provider_annas_archive.service import (
+    CATALOG_FALLBACK_DOMAINS,
     DEFAULT_OFFICIAL_URL,
     SUPPORTED_OFFICIAL_DOMAINS,
     SUPPORTED_OFFICIAL_URLS,
@@ -76,7 +77,7 @@ async def manifest() -> ManifestResponse:
             homepage_url="https://github.com/pullboxapp/pullbox-direct-providers",
             documentation_url="https://github.com/pullboxapp/pullbox-direct-providers",
             support_url="https://github.com/pullboxapp/pullbox-direct-providers/issues",
-            source_domains=list(SUPPORTED_OFFICIAL_DOMAINS),
+            source_domains=[*SUPPORTED_OFFICIAL_DOMAINS, *CATALOG_FALLBACK_DOMAINS],
             artifact_host_patterns=["generic_https"],
             capabilities=ProviderCapabilities(
                 search=True,

providers/annas_archive/src/pullbox_provider_annas_archive/service.py

@@ -6,7 +6,7 @@
 import re
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
 from urllib.parse import urlencode, urlsplit
 
 import httpx
@@ -22,6 +22,7 @@
 )
 from pullbox_provider_contract.search_terms import collection_title_fragment, is_collection_intent
 from pullbox_provider_contract.source_http import fetch_source_html
+from pullbox_provider_libgen.service import KNOWN_SOURCE_DOMAINS, LibGenProviderService
 
 from pullbox_provider_annas_archive.parser import parse_search_html
 
@@ -34,6 +35,7 @@
     "annas-archive.gd",
 )
 SUPPORTED_OFFICIAL_URLS = tuple(f"https://{domain}" for domain in SUPPORTED_OFFICIAL_DOMAINS)
+CATALOG_FALLBACK_DOMAINS = KNOWN_SOURCE_DOMAINS
 DEFAULT_OFFICIAL_URL = "https://annas-archive.gd"
 _MD5 = re.compile(r"\A[a-f0-9]{32}\Z")
 _MAX_JSON_BYTES = 256 * 1024
@@ -44,6 +46,17 @@
 FastDownloadFetcher = Callable[..., Awaitable[tuple[int, dict[str, object]]]]
 
 
+class CatalogFallback(Protocol):
+    async def search(
+        self,
+        intent: SearchIntent,
+        *,
+        provider_config: Mapping[str, object],
+        limit: int,
+        resolver_profile: ResolverProfile | None = None,
+    ) -> list[Candidate]: ...
+
+
 @dataclass(frozen=True, slots=True)
 class AnnasArchiveResolveResult:
     """Resolved artifacts plus safe source-account capacity telemetry."""
@@ -60,9 +73,11 @@ def __init__(
         *,
         page_fetcher: PageFetcher = fetch_source_html,
         fast_download_fetcher: FastDownloadFetcher | None = None,
+        catalog_fallback: CatalogFallback | None = None,
     ) -> None:
         self._page_fetcher = page_fetcher
         self._fast_download_fetcher = fast_download_fetcher or _fetch_fast_download
+        self._catalog_fallback = catalog_fallback or LibGenProviderService()
 
     async def search(
         self,
@@ -75,15 +90,43 @@ async def search(
         domain = validate_official_domain(str(provider_config.get("domain", DEFAULT_OFFICIAL_URL)))
         query = _build_query(intent)
         params = urlencode([("q", query), ("ext", "cbz"), ("ext", "cbr"), ("ext", "pdf")])
-        html = await self._page_fetcher(
-            f"{domain}/search?{params}",
-            declared_domains=((urlsplit(domain).hostname or ""),),
-            resolver_profile=resolver_profile,
+        primary_error: RuntimeError | None = None
+        try:
+            html = await self._page_fetcher(
+                f"{domain}/search?{params}",
+                declared_domains=((urlsplit(domain).hostname or ""),),
+                resolver_profile=resolver_profile,
+            )
+            candidates = parse_search_html(
+                html,
+                source_domain=urlsplit(domain).hostname or "annas-archive.gd",
+            )[:limit]
+        except asyncio.CancelledError:
+            raise
+        except RuntimeError as exc:
+            primary_error = exc
+        else:
+            if candidates:
+                return candidates
+
+        try:
+            catalog_candidates = await self._catalog_fallback.search(
+                intent,
+                provider_config={},
+                limit=limit,
+                resolver_profile=resolver_profile,
+            )
+        except asyncio.CancelledError:
+            raise
+        except (RuntimeError, ValueError):
+            if primary_error is not None:
+                raise primary_error from None
+            return []
+        return _anna_candidates_from_catalog(
+            catalog_candidates,
+            domain=domain,
+            limit=limit,
         )
-        return parse_search_html(
-            html,
-            source_domain=urlsplit(domain).hostname or "annas-archive.gd",
-        )[:limit]
 
     async def resolve(
         self,
@@ -293,6 +336,52 @@ def _build_query(intent: SearchIntent) -> str:
     return " ".join(parts)[:700]
 
 
+def _anna_candidates_from_catalog(
+    catalog_candidates: Sequence[Candidate],
+    *,
+    domain: str,
+    limit: int,
+) -> list[Candidate]:
+    candidates: list[Candidate] = []
+    seen: set[str] = set()
+    for candidate in sorted(catalog_candidates, key=_catalog_candidate_rank):
+        rank = _catalog_candidate_rank(candidate)
+        fingerprint = candidate.content_fingerprint or ""
+        md5 = fingerprint.removeprefix("md5:") if fingerprint.startswith("md5:") else ""
+        if (
+            not _MD5.fullmatch(md5)
+            or candidate.provider_candidate_id != f"libgen:{md5}"
+            or md5 in seen
+        ):
+            continue
+        seen.add(md5)
+        candidates.append(
+            candidate.model_copy(
+                update={
+                    "provider_candidate_id": f"anna:{md5}",
+                    "source_reference": f"{domain}/md5/{md5}",
+                    # Catalog presence does not prove that Anna exposes a
+                    # member fast-download route for this exact LibGen file.
+                    "content_fingerprint": None,
+                    "provider_confidence": max(candidate.provider_confidence - (0.05 * rank), 0),
+                    "provenance": {
+                        "layout": "libgen-catalog-fallback-v1",
+                        "source_kind": "metadata",
+                        "catalog_source": "libgen",
+                    },
+                }
+            )
+        )
+        if len(candidates) == limit:
+            break
+    return candidates
+
+
+def _catalog_candidate_rank(candidate: Candidate) -> int:
+    """Prefer canonical files over mobile derivatives while keeping source order."""
+    return 1 if "digital-mobile" in candidate.display_title.casefold() else 0
+
+
 def _safe_download_url(raw_url: str) -> bool:
     return _safe_download_origin(raw_url) is not None
 
@@ -367,7 +456,10 @@ def _is_quota_error(payload: Mapping[str, object]) -> bool:
 
 def _is_candidate_unavailable(payload: Mapping[str, object]) -> bool:
     error = payload.get("error")
-    return isinstance(error, str) and "invalid domain_index or path_index" in error.casefold()
+    return isinstance(error, str) and any(
+        marker in error.casefold()
+        for marker in ("invalid domain_index or path_index", "record not found")
+    )
 
 
 def _quota_status(payload: Mapping[str, object]) -> QuotaStatus | None:

tests/unit/test_annas_archive_app.py

@@ -106,6 +106,11 @@ async def test_anna_manifest_marks_member_key_secret_and_official_urls_editable(
         "annas-archive.gl",
         "annas-archive.pk",
         "annas-archive.gd",
+        "libgen.gl",
+        "libgen.li",
+        "libgen.vg",
+        "libgen.la",
+        "libgen.bz",
     ]
     assert payload["artifact_host_patterns"] == ["generic_https"]
     schema = payload["configuration_schema"]