free_web_scraper.py

"""Free web scraper: fetch any URL with plain requests and extract html, text or JSON.

This is the do-it-yourself version of the same job the API does. No key, no cost.
For a large share of the web it is all you need, and this repo will tell you when
that is the case rather than selling you something you do not have to buy.

    pip install requests beautifulsoup4
    python free_scraper/free_web_scraper.py https://example.com
    python free_scraper/free_web_scraper.py https://api.github.com/repos/python/cpython
    python free_scraper/free_web_scraper.py https://www.ebay.com/sch/i.html?_nkw=laptop --parse text

It parses first and only reports a block when the data is genuinely absent. A page
is not "blocked" because the word captcha appears somewhere in its JavaScript;
plenty of working pages ship anti-bot script on a perfectly good response.
"""
import argparse
import json
import re
import sys

import requests

UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")

# Markers scoped to the <title> and the first few KB only. A marker deep inside a
# 600 KB body is almost always a false positive from the site's own anti-bot script
# running on a page that loaded fine.
BLOCK_TITLES = re.compile(
    r"(robot or human|are you a robot|access denied|access to this page has been denied"
    r"|please wait|just a moment|attention required|error page|blocked|forbidden"
    r"|verify you are human|security check)",
    re.I,
)


def fetch(url: str, timeout: int = 45) -> requests.Response:
    return requests.get(
        url,
        headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9",
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"},
        timeout=timeout,
    )


def title_of(html: str):
    m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S)
    return re.sub(r"\s+", " ", m.group(1)).strip() if m else None


def to_text(html: str) -> str:
    try:
        from bs4 import BeautifulSoup
    except ImportError:
        sys.exit("pip install beautifulsoup4")
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "noscript"]):
        tag.decompose()
    return re.sub(r"\s+", " ", soup.get_text(" ")).strip()


def extract(body: str, mode: str) -> tuple:
    """Return (content_type, payload). Mirrors what the API's parse param does."""
    stripped = body.lstrip()
    looks_json = stripped.startswith("{") or stripped.startswith("[")
    if mode == "json" or (mode == "auto" and looks_json):
        try:
            return "json", json.loads(body)
        except ValueError:
            pass  # not JSON after all: fall through to html, same as the API
    if mode == "text":
        return "text", to_text(body)
    return "html", body


def looks_blocked(r: requests.Response, content_type: str) -> str:
    """Only call it a block when the data is genuinely absent. Parse first.

    Judged against the raw body whatever the parse mode, since that is where the
    evidence lives. Returns a reason string, or "" when this looks like content.
    """
    # A body that parsed as JSON is real data by definition. Never a block.
    if content_type == "json":
        return ""
    body = r.text
    title = title_of(body) or ""
    if r.status_code in (401, 403, 429) or r.status_code >= 500:
        return f"HTTP {r.status_code}" + (f", title {title!r}" if title else "")
    if BLOCK_TITLES.search(title):
        return f"HTTP {r.status_code} but the title is {title!r}"
    # A real content page is essentially never this small. Scope the marker check to
    # the head of the document so a working page's anti-bot script cannot trip it.
    if len(body) < 20_000 and BLOCK_TITLES.search(body[:4000]):
        return f"HTTP {r.status_code}, only {len(body):,} chars, block marker in the head"
    return ""


def main() -> int:
    ap = argparse.ArgumentParser(description="Fetch any URL with plain requests.")
    ap.add_argument("url")
    ap.add_argument("--parse", default="auto", choices=["auto", "html", "text", "json"])
    ap.add_argument("--max", type=int, default=1200, help="chars of payload to print")
    a = ap.parse_args()

    try:
        r = fetch(a.url)
    except requests.exceptions.RequestException as e:
        print(f"FAILED: {type(e).__name__}: {str(e)[:120]}")
        return 2

    content_type, payload = extract(r.text, a.parse)
    reason = looks_blocked(r, content_type)

    print(f"url          : {a.url}")
    print(f"http_status  : {r.status_code}")
    print(f"content_type : {content_type}")
    print(f"chars        : {len(r.text):,}")   # decoded characters, not the compressed transfer
    if content_type != "json":
        print(f"title        : {title_of(r.text)}")

    if reason:
        print()
        print(f"BLOCKED: {reason}.")
        if r.status_code == 200:
            print("A 200 does not mean success: this page has no usable content in it.")
        print("Nothing in your code can fix this: it is decided before your parser runs.")
        return 1

    print()
    if content_type == "json":
        out = json.dumps(payload, indent=2)[:a.max]
    else:
        out = (payload if isinstance(payload, str) else str(payload))[:a.max]
    print(out)
    print()
    size = len(json.dumps(payload)) if content_type == "json" else len(payload)
    print(f"OK: extracted {size:,} chars as {content_type}.")
    return 0


if __name__ == "__main__":
    sys.exit(main())