added static analysis and graph zoom, and full screen mode

Author: lorenzo-moni

.gitignore

@@ -1,3 +1,6 @@
 venv
 .env
-tmp
+tmp
+
+__pycache__
+.DS_Store

dataset/.DS_Store

@@ -16,7 +16,6 @@
 import pandas as pd
 from rapidfuzz import fuzz, process
 
-
 RAWG_NAME_COL = "name"
 RAWG_DATE_COL = "released"
 STEAM_ID_COL = "appID"
@@ -44,6 +43,29 @@
     "required_age",
 ]
 
+# Manual overrides: RAWG name → Steam appID.
+# Use this when the game was renamed, replaced, or the date mismatch is too large
+# for the fuzzy passes to bridge (e.g. CS:GO → Counter-Strike 2, same appID 730).
+MANUAL_OVERRIDES: dict[str, str] = {
+    "Counter-Strike: Global Offensive": "730",
+    "Total War: SHOGUN 2": "34330",
+    "Blasphemous 2": "2114740",
+    "Batman: Arkham City": "200260",
+    "Grand Theft Auto V": "271590",
+    "PlayerUnknown’s Battlegrounds": "578080",
+    "Monster Hunter Wilds": "2246340",
+    "Apex Legends": "1172470",
+    "ARK: Survival Evolved": "346110",
+    "Garry's Mod": "4000",
+    "DayZ": "221100",
+    "7 Days to Die": "251570",
+    "Helldivers 2": "553850",
+    "Anthem": "2656490",
+    "Battlefield 2042": "1517290",
+    "Fall Guys": "1097150",
+    "Destiny 2": "1085660",
+}
+
 EDITION_RE = re.compile(
     r"\b(goty|game of the year|definitive|remastered|remaster|deluxe|"
     r"complete|ultimate|enhanced|anniversary|collectors?|standard|"
@@ -107,14 +129,25 @@ def match_rawg_steam(rawg_df: pd.DataFrame, steam_df: pd.DataFrame) -> pd.DataFr
     )
     print(f"[join] RAWG rows: {len(rawg_df)}  |  Steam clean rows: {len(steam_clean)}")
 
-    # Pass 1
+    # Pass 0 — manual overrides (RAWG name → Steam appID)
+    rawg_df["steam_appid"] = rawg_df[RAWG_NAME_COL].map(MANUAL_OVERRIDES)
+    rawg_df["match_type"] = None
+    rawg_df.loc[rawg_df["steam_appid"].notna(), "match_type"] = "manual"
+    print(f"[join] Pass 0:  {(rawg_df['match_type'] == 'manual').sum():>6}")
+
+    # Pass 1 — exact key+year on still-unmatched rows
     p1 = rawg_df.merge(
-        steam_clean[[STEAM_ID_COL, "key", "year"]],
+        steam_clean[[STEAM_ID_COL, "key", "year"]].rename(
+            columns={STEAM_ID_COL: "_steam_p1"}
+        ),
         on=["key", "year"],
         how="left",
-    ).rename(columns={STEAM_ID_COL: "steam_appid"})
-    p1["match_type"] = p1["steam_appid"].notna().map({True: "exact", False: None})
-    print(f"[join] Pass 1:  {p1['steam_appid'].notna().sum():>6}")
+    )
+    new_exact = p1["steam_appid"].isna() & p1["_steam_p1"].notna()
+    p1.loc[new_exact, "steam_appid"] = p1.loc[new_exact, "_steam_p1"]
+    p1.loc[new_exact, "match_type"] = "exact"
+    p1 = p1.drop(columns=["_steam_p1"])
+    print(f"[join] Pass 1:  {(p1['match_type'] == 'exact').sum():>6}")
 
     #  Pass 2
     missing_mask = p1["steam_appid"].isna()

dataset/processing/__pycache__/join.cpython-314.pyc

@@ -44,20 +44,20 @@ def run():
 
     # Step 3: Scrape Steamcharts for every matched Steam app ID
     print("STEP 3: Scraping Steamcharts time series")
-    games_dict = (
+    _df = (
         matched_df[matched_df["steam_appid"].notna()]
         .drop_duplicates(subset=["steam_appid"])
         .assign(appid_int=lambda d: pd.to_numeric(d["steam_appid"], errors="coerce"))
         .dropna(subset=["appid_int"])
-        .set_index("appid_int")["name"]
-        .rename(index=int)
-        .to_dict()
+        .query("ratings_count >= 30")
     )
-    # Keys must be plain Python ints for the URL builder
-    games_dict = {int(k): v for k, v in games_dict.items()}
+    games_dict = {
+        int(r["appid_int"]): (r["name"], float(r["ratings_count"] or 0))
+        for _, r in _df.iterrows()
+    }
 
     # Scrape game by game, append immediately to a temp CSV
-    scrape_to_disk(games_dict, TMP_STEAMCHARTS)
+    scrape_to_disk(games_dict, TMP_STEAMCHARTS, 0.5)
     sc_df = build_timeseries(TMP_STEAMCHARTS)
 
     # Step 4: Final intersection to keep only games in all three sources

dataset/processing/__pycache__/rawg.cpython-314.pyc

@@ -13,32 +13,17 @@
 import os
 import time
 from io import StringIO
-
-import pandas as pd
-import requests
 import random
 
-
-USER_AGENTS = [
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15",
-    "Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0",
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36",
-    "Mozilla/5.0 (iPhone; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1",
-]
+import pandas as pd
+from curl_cffi import requests
 
 
 def fetch_steamcharts(app_id: int, name: str) -> pd.DataFrame | None:
     """Fetch monthly player count history for a single game"""
     url = f"https://steamcharts.com/app/{app_id}"
-    headers = {
-        "User-Agent": random.choice(USER_AGENTS),
-    }
     try:
-        r = requests.get(url, headers=headers, timeout=15)
+        r = requests.get(url, impersonate="chrome136", timeout=15)
         if r.status_code != 200:
             print(f"{name}: HTTP {r.status_code}")
             return None
@@ -75,17 +60,25 @@ def scrape_to_disk(games: dict, output_path: str, delay: float = 4) -> int:
     Number of new rows written in this run.
     """
     os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
+    failed_path = output_path.replace(".csv", "_failed.csv")
 
     safely_done: set[int] = set()
+    failed_ids: set[int] = set()
     header_written = False
 
+    # Load permanently failed IDs from previous runs
+    if os.path.exists(failed_path):
+        failed_df = pd.read_csv(failed_path)
+        failed_ids = set(failed_df["app_id"].dropna().astype(int).tolist())
+        print(f"[steamcharts] Skipping {len(failed_ids)} permanently failed app_ids")
+
     if os.path.exists(output_path):
         try:
             existing = pd.read_csv(output_path)
             all_ids = existing["app_id"].dropna().astype(int).tolist()
             if all_ids:
                 last_id = all_ids[-1]
-                # Drop the last game's rows because it they may be truncated
+                # Drop the last game's rows because they may be truncated
                 existing_clean = existing[existing["app_id"].astype(int) != last_id]
                 existing_clean.to_csv(output_path, index=False)
                 safely_done = set(
@@ -99,14 +92,18 @@ def scrape_to_disk(games: dict, output_path: str, delay: float = 4) -> int:
         except Exception:
             pass
 
-    remaining = {k: v for k, v in games.items() if int(k) not in safely_done}
+    remaining = {
+        k: v
+        for k, v in sorted(games.items(), key=lambda x: x[1][1], reverse=True)
+        if int(k) not in safely_done and int(k) not in failed_ids
+    }
     print(f"[steamcharts] {len(remaining)} games to scrape: {output_path}")
 
     rows_written = 0
     fetched = 0
 
-    for app_id, name in remaining.items():
-        print(f"  ({len(safely_done) + fetched + 1}/{len(games)}) {name}...")
+    for app_id, (name, _) in remaining.items():
+        print(f"  ({len(safely_done) + fetched + 1}/{len(games)}) {name} {app_id}...")
         df_game = fetch_steamcharts(int(app_id), name)
         if df_game is not None:
             df_game.to_csv(
@@ -118,7 +115,14 @@ def scrape_to_disk(games: dict, output_path: str, delay: float = 4) -> int:
             rows_written += len(df_game)
             header_written = True
             fetched += 1
-        time.sleep(delay)
+        else:
+            pd.DataFrame({"app_id": [int(app_id)], "name": [name]}).to_csv(
+                failed_path,
+                mode="a",
+                index=False,
+                header=not os.path.exists(failed_path),
+            )
+        time.sleep(delay + random.uniform(0, 2))
 
     print(
         f"[steamcharts] Done: {fetched} new games fetched, {rows_written} new rows written"