refactor: modularize and restructure cynicmansion parser

Author: gil9red

cynicmansion/common.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+__author__ = "ipetrash"
+
+
+from urllib.parse import urljoin
+
+# pip install requests==2.32.2
+import requests
+
+# pip install beautifulsoup4==4.15.0
+from bs4 import BeautifulSoup
+
+HOST: str = "https://cynicmansion.com"
+
+session = requests.session()
+session.headers.update(
+    {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:70.0) Gecko/20100101 Firefox/70.0",
+        "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
+    }
+)
+
+
+def get_comics_url_by_id(comics_id: str | int) -> str:
+    return f"{HOST}/{comics_id}/"
+
+
+def get_comics_info(
+    comics_id__or__url: str | int,
+) -> tuple[str, str, str]:  # TODO: dataclass
+    url_comics: str
+    if isinstance(comics_id__or__url, str) and comics_id__or__url.startswith("http"):
+        url_comics = comics_id__or__url
+    else:
+        url_comics = get_comics_url_by_id(comics_id__or__url)
+
+    rs = session.get(url_comics)
+    rs.raise_for_status()
+
+    root = BeautifulSoup(rs.content, "html.parser")
+
+    title = root.select_one(".comics_name").text.strip()
+
+    url_image = urljoin(rs.url, root.select_one(".comics_image > img")["src"])
+
+    return url_comics, title, url_image
+
+
+def get_last_commics_id() -> int:
+    rs = session.get(HOST)
+    rs.raise_for_status()
+
+    root = BeautifulSoup(rs.content, "html.parser")
+
+    # Example: <a href="/2184/">комментарии (1)</a> -> 2184
+    return int(
+        root.select_one(
+            ".comics_wrap:has(.comics_name) table > tr > td:first-child > a[href]"
+        )["href"].replace("/", "")
+    )
+
+
+def get_comics_image_url(comics_id__or__url: str | int) -> str:
+    _, _, url_image = get_comics_info(comics_id__or__url)
+    return url_image
+
+
+if __name__ == "__main__":
+    last_commics_id = get_last_commics_id()
+    print("last_commics_id:", last_commics_id)
+
+    print()
+    comics_id = 1538
+    print(f"About #{comics_id} comics")
+
+    url = get_comics_url_by_id(comics_id)
+    print(comics_id, url)
+    print()
+    print(get_comics_info(comics_id))
+    print(get_comics_info(url))
+    print()
+    print(get_comics_image_url(comics_id))
+    print(get_comics_image_url(url))

cynicmansion/download_all.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+__author__ = "ipetrash"
+
+
+import time
+import re
+
+from pathlib import Path
+
+from common import session, get_last_commics_id, get_comics_info
+
+CURRENT_DIR: Path = Path(__file__).resolve().parent
+DOWNLOADS_DIR: Path = CURRENT_DIR / "downloads"
+DOWNLOADS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def download_all(downloads_dir: Path = DOWNLOADS_DIR) -> None:
+    comics_number = get_last_commics_id()
+    for comics_id in range(1, comics_number + 1):
+        url_comics, title, url_image = get_comics_info(comics_id)
+        print(f"#{comics_id}. {title!r}: {url_image}")
+
+        clear_title: str = re.sub(r'[^\w\- ]', '', title)
+        file_name: Path = downloads_dir / f"{comics_id} - {clear_title}.jpg"
+        print(f"Saving to: {file_name}")
+
+        # TODO: Повторяет логику из get_random_comics.py
+        with open(file_name, mode="wb") as f:
+            rs = session.get(url_image)
+            rs.raise_for_status()
+
+            f.write(rs.content)
+
+        print()
+
+        time.sleep(1)
+
+
+if __name__ == '__main__':
+    download_all()

cynicmansion/get_random_comics.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+__author__ = "ipetrash"
+
+
+import random
+
+from common import (
+    session,
+    get_comics_image_url,
+    get_last_commics_id,
+    get_comics_url_by_id,
+    get_comics_info,
+)
+
+
+def get_random_comics_url() -> str:
+    comics_number = get_last_commics_id()
+
+    random_comics_number = random.randint(1, comics_number)
+    return get_comics_url_by_id(random_comics_number)
+
+
+def get_random_comics_image_url() -> str:
+    url = get_random_comics_url()
+
+    return get_comics_image_url(url)
+
+
+def get_random_comics_info() -> tuple[str, str, str]:
+    url = get_random_comics_url()
+
+    return get_comics_info(url)
+
+
+if __name__ == "__main__":
+    url_comics = get_random_comics_url()
+    print("url_comics:", url_comics)
+
+    url_image = get_comics_image_url(url_comics)
+    print("url_image:", url_image)
+
+    comics_id = url_comics.split("/")[-2]
+
+    from pathlib import Path
+
+    current_dir: Path = Path(__file__).resolve().parent
+    downloads_dir: Path = current_dir / "downloads"
+    downloads_dir.mkdir(parents=True, exist_ok=True)
+
+    file_name = downloads_dir / f"{comics_id}.jpg"
+    print(f"Saving to: {file_name}")
+
+    # TODO: Повторяет логику из download_all.py
+    with open(file_name, mode="wb") as f:
+        rs = session.get(url_image)
+        rs.raise_for_status()
+
+        f.write(rs.content)
+
+    print()
+    print("Random comics image:")
+    print(get_random_comics_image_url())
+    print(get_random_comics_image_url())
+    print(get_random_comics_image_url())
+
+    print()
+    print("Random comics image:")
+    print(get_random_comics_info())
+    print(get_random_comics_info())
+    print(get_random_comics_info())