update

Author: erkanrzgc

core/http_client.py

@@ -37,6 +37,80 @@
 log = get_logger(__name__)
 
 
+# ── Scrapling stealth transport (optional) ────────────────────────────
+_SCRAPLING_AVAILABLE = False
+_SCRAPLING_FETCHER = None
+_SCRAPLING_INITED = False
+
+try:
+    from scrapling import AsyncFetcher  # type: ignore[import-not-found]
+
+    _SCRAPLING_AVAILABLE = True
+except ImportError:
+    pass
+
+
+async def _init_scrapling():
+    global _SCRAPLING_INITED, _SCRAPLING_FETCHER
+    if _SCRAPLING_INITED:
+        return _SCRAPLING_FETCHER
+    _SCRAPLING_INITED = True
+    if not _SCRAPLING_AVAILABLE:
+        return None
+    try:
+        _SCRAPLING_FETCHER = AsyncFetcher()
+        log.debug("Scrapling transport active")
+        return _SCRAPLING_FETCHER
+    except Exception as exc:
+        log.debug("Scrapling init failed: %s", exc)
+        return None
+
+
+async def _try_scrapling_get(url, headers, timeout):
+    """Try Scrapling fetch. Returns None on failure (use aiohttp)."""
+    if not _SCRAPLING_AVAILABLE:
+        return None
+    if not url.startswith("https://"):
+        return None
+    if any(d in url for d in ("example.com", "fake.test", "localhost", "127.0.0.1")):
+        return None
+    fetcher = await _init_scrapling()
+    if fetcher is None:
+        return None
+    try:
+        t0 = time.monotonic()
+        resp = await asyncio.wait_for(
+            fetcher.get(url, headers=headers),
+            timeout=timeout,
+        )
+        elapsed = time.monotonic() - t0
+        return resp.status, resp.html_content, elapsed, resp.url
+    except asyncio.TimeoutError:
+        return 0, "", time.monotonic() - t0, None
+    except Exception as exc:
+        log.debug("Scrapling fetch failed for %s: %s", url, exc)
+        return None
+
+
+def _should_retry_scrapling(status, body, url):
+    """Check if aiohttp response looks blocked — worth a Scrapling retry."""
+    if not _SCRAPLING_AVAILABLE:
+        return False
+    if status in (403, 429, 503):
+        return True
+    if status == 200 and body and len(body) < 2000:
+        blocked = any(s in body for s in (
+            "Just a moment",
+            "cf-browser-verification",
+            "Attention Required",
+            "captcha",
+            "_cf_chl_opt",
+        ))
+        if blocked:
+            return True
+    return False
+
+
 def _backoff(attempt: int) -> float:
     """Exponential backoff with jitter to avoid thundering-herd on mass retries."""
     return RETRY_DELAY * (2 ** attempt) * random.uniform(0.5, 1.5)
@@ -194,9 +268,15 @@ async def get(
         headers: dict | None = None,
         allow_redirects: bool = True,
     ) -> tuple[int, str, float]:
-        status, body, elapsed, _ = await self._get_internal(
+        status, body, elapsed, final = await self._get_internal(
             url, headers, allow_redirects=allow_redirects
         )
+        # When aiohttp gets blocked (403 / empty body / CF challenge),
+        # retry via Scrapling's stealthier TLS transport.
+        if _should_retry_scrapling(status, body, url):
+            sr = await _try_scrapling_get(url, headers, self._request_timeout)
+            if sr is not None:
+                return sr[0], sr[1], sr[2]
         return status, body, elapsed
 
     async def get_with_meta(

core/scrapling_client.py

@@ -0,0 +1,143 @@
+"""Scrapling-based HTTP client adapter.
+
+Provides the same interface as HTTPClient (get, get_json, post_json) but uses
+Scrapling's curl_cffi-backed transport, which impersonates Chrome's TLS
+fingerprint and bypasses many bot-detection systems that block plain aiohttp.
+
+Drop-in replacement::
+
+    from core.scrapling_client import ScraplingClient
+    async with ScraplingClient() as client:
+        status, body, elapsed = await client.get(url)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json as _json
+import logging
+import time
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+try:
+    from scrapling import AsyncFetcher
+
+    SCRAPLING_AVAILABLE = True
+except ImportError:  # pragma: no cover
+    AsyncFetcher = None  # type: ignore[assignment]
+    SCRAPLING_AVAILABLE = False
+
+
+class ScraplingClient:
+    """Async HTTP client backed by Scrapling's stealthy TLS transport."""
+
+    def __init__(
+        self,
+        *,
+        proxy: str | None = None,
+        request_timeout: float = 15.0,
+        fingerprint: bool = True,
+    ) -> None:
+        self._proxy = proxy
+        self._timeout = request_timeout
+        self._fingerprint = fingerprint
+        self._fetcher: Any = None
+
+    async def __aenter__(self) -> ScraplingClient:
+        if not SCRAPLING_AVAILABLE:
+            raise RuntimeError("Scrapling is not installed")
+        self._fetcher = AsyncFetcher()
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        self._fetcher = None
+
+    async def get(
+        self,
+        url: str,
+        headers: dict[str, str] | None = None,
+    ) -> tuple[int, str, float]:
+        """Fetch *url* and return ``(status, body, elapsed)``."""
+        t0 = time.monotonic()
+        fetcher = self._fetcher
+        if fetcher is None:
+            return -1, "", 0.0
+        try:
+            resp = await asyncio.wait_for(
+                fetcher.get(url, headers=headers),
+                timeout=self._timeout,
+            )
+            elapsed = time.monotonic() - t0
+            return resp.status, resp.html_content, elapsed
+        except asyncio.TimeoutError:
+            return 0, "", time.monotonic() - t0
+        except Exception as exc:
+            log.debug("Scrapling get failed for %s: %s", url, exc)
+            return -1, "", time.monotonic() - t0
+
+    async def get_json(
+        self,
+        url: str,
+        headers: dict[str, str] | None = None,
+    ) -> tuple[int, dict | None, float]:
+        """Fetch JSON from *url* and return ``(status, parsed, elapsed)``."""
+        t0 = time.monotonic()
+        fetcher = self._fetcher
+        if fetcher is None:
+            return -1, None, 0.0
+        try:
+            resp = await asyncio.wait_for(
+                fetcher.get(url, headers=headers),
+                timeout=self._timeout,
+            )
+            elapsed = time.monotonic() - t0
+            if resp.status != 200:
+                return resp.status, None, elapsed
+            try:
+                return resp.status, resp.json(), elapsed
+            except Exception:
+                try:
+                    return resp.status, _json.loads(resp.html_content), elapsed
+                except Exception:
+                    return resp.status, None, elapsed
+        except asyncio.TimeoutError:
+            return 0, None, time.monotonic() - t0
+        except Exception as exc:
+            log.debug("Scrapling get_json failed for %s: %s", url, exc)
+            return -1, None, time.monotonic() - t0
+
+    async def post_json(
+        self,
+        url: str,
+        json_body: dict,
+        headers: dict[str, str] | None = None,
+    ) -> tuple[int, dict | None, float]:
+        """POST JSON and return ``(status, parsed, elapsed)``."""
+        t0 = time.monotonic()
+        fetcher = self._fetcher
+        if fetcher is None:
+            return -1, None, 0.0
+        merged = dict(headers or {})
+        merged.setdefault("Content-Type", "application/json")
+        try:
+            resp = await asyncio.wait_for(
+                fetcher.post(url, json=json_body, headers=merged),
+                timeout=self._timeout,
+            )
+            elapsed = time.monotonic() - t0
+            if resp.status != 200:
+                return resp.status, None, elapsed
+            try:
+                return resp.status, resp.json(), elapsed
+            except Exception:
+                try:
+                    return resp.status, _json.loads(resp.html_content), elapsed
+                except Exception:
+                    return resp.status, None, elapsed
+        except asyncio.TimeoutError:
+            return 0, None, time.monotonic() - t0
+        except Exception as exc:
+            log.debug("Scrapling post failed for %s: %s", url, exc)
+            return -1, None, time.monotonic() - t0