fix(download): fix the concurrency on osmsg

add the about section too

Author: kshitijrajsharma

osmsg/_http.py

@@ -11,7 +11,7 @@
 from urllib3.util.retry import Retry
 
 USER_AGENT = "osmsg"
-DEFAULT_TIMEOUT = (10, 60)  # (connect, read) seconds
+DEFAULT_TIMEOUT = (30, 120)  # (connect, read) seconds
 
 
 class _TimeoutSession(requests.Session):
@@ -26,10 +26,14 @@ def make_session() -> requests.Session:
     """Fresh session with the standard timeout + retry policy (use when a flow needs its own cookie jar)."""
     s = _TimeoutSession()
     retry = Retry(
-        total=5,
-        backoff_factor=0.5,
+        total=10,
+        connect=10,
+        read=10,
+        backoff_factor=1.0,
+        backoff_max=120,
         status_forcelist=(429, 500, 502, 503, 504),
         allowed_methods=frozenset({"GET", "POST", "HEAD"}),
+        respect_retry_after_header=True,
     )
     adapter = HTTPAdapter(max_retries=retry, pool_maxsize=32)
     s.mount("https://", adapter)

osmsg/gui.py

@@ -7,14 +7,21 @@
 import queue
 import sys
 import threading
+import webbrowser
 from pathlib import Path
 from typing import Any
 
+from .__version__ import __version__
 from .exceptions import NoDataFoundError, OsmsgError
 from .pipeline import RunConfig, run
 
 UTC = dt.UTC
 FORMATS = ["parquet", "csv", "json", "markdown"]
+ABOUT_LINKS = [
+    ("Star osmsg on GitHub", "https://github.com/osgeonepal/osmsg"),
+    ("Report a bug or request a feature", "https://github.com/osgeonepal/osmsg/issues"),
+    ("Sponsor the developer", "https://github.com/sponsors/kshitijrajsharma"),
+]
 PRESETS = ["Last hour", "Last day", "Last week", "Last month", "Last year", "All time"]
 _PRESET_DELTAS = {
     "Last hour": dt.timedelta(hours=1),
@@ -109,6 +116,7 @@ def __init__(self) -> None:
         from tkinter import filedialog, scrolledtext, ttk
 
         self._tk = tk
+        self._ttk = ttk
         self._filedialog = filedialog
         self.events: queue.Queue = queue.Queue()
         self.out_dir = str(Path.home() / "osmsg")
@@ -164,8 +172,26 @@ def __init__(self) -> None:
 
         self.log = scrolledtext.ScrolledText(frame, width=70, height=14, state="disabled")
         self.log.grid(row=10, column=0, columnspan=4, sticky="nsew")
+
+        ttk.Button(frame, text="About", command=self._show_about).grid(row=11, column=0, pady=(6, 0), sticky="w")
+        ttk.Label(frame, text="A project of OSGeo Nepal").grid(row=11, column=1, columnspan=3, pady=(6, 0), sticky="e")
         self.root.after(120, self._drain)
 
+    def _show_about(self) -> None:
+        tk, ttk = self._tk, self._ttk
+        win = tk.Toplevel(self.root)
+        win.title("About osmsg")
+        box = ttk.Frame(win, padding=16)
+        box.grid(sticky="nsew")
+        ttk.Label(box, text=f"osmsg {__version__}", font=("", 12, "bold")).grid(sticky="w")
+        ttk.Label(box, text="OpenStreetMap Stats Generator").grid(sticky="w")
+        ttk.Label(box, text="A project of OSGeo Nepal").grid(sticky="w", pady=(0, 10))
+        for text, url in ABOUT_LINKS:
+            link = ttk.Label(box, text=text, foreground="#1a73e8", cursor="hand2")
+            link.grid(sticky="w", pady=2)
+            link.bind("<Button-1>", lambda _event, target=url: webbrowser.open(target))
+        ttk.Button(box, text="Close", command=win.destroy).grid(sticky="e", pady=(12, 0))
+
     def _apply_preset(self, name: str) -> None:
         start, end = preset_range(name)
         self.vars["start"].set(_fmt(start))

osmsg/pipeline.py

@@ -14,6 +14,7 @@
 from typing import Any
 
 import duckdb
+import requests
 from platformdirs import user_cache_dir
 from shapely.ops import unary_union
 
@@ -539,6 +540,11 @@ def _processing_config(cfg: RunConfig, *, parquet_dir: Path, geom_wkt: str | Non
     }
 
 
+# Replication servers throttle many concurrent connections, so downloads stay polite regardless of
+# the worker count used for local parsing. Already-downloaded files are cached, so a rerun resumes.
+_DOWNLOAD_WORKERS = 4
+
+
 def _download_all(
     urls: list[str],
     mode: str,
@@ -548,12 +554,19 @@ def _download_all(
     label: str,
     description: str = "downloading",
 ) -> None:
-    with (
-        progress_bar(len(urls), unit=label, description=description) as advance,
-        concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool,
-    ):
-        for _ in pool.map(lambda u: download_osm_file(u, mode=mode, cookie=cookie, cache_dir=cache_dir), urls):
-            advance()
+    workers = min(max_workers, _DOWNLOAD_WORKERS)
+    try:
+        with (
+            progress_bar(len(urls), unit=label, description=description) as advance,
+            concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool,
+        ):
+            for _ in pool.map(lambda u: download_osm_file(u, mode=mode, cookie=cookie, cache_dir=cache_dir), urls):
+                advance()
+    except requests.exceptions.RequestException as exc:
+        raise OsmsgError(
+            f"Network error downloading {label} after retries ({type(exc).__name__}). "
+            "Re-run to resume: finished downloads are cached, so it continues from where it stopped."
+        ) from exc
 
 
 def _process_all(
@@ -727,6 +740,13 @@ def run(cfg: RunConfig) -> dict[str, Any]:
             else f"first run with {cfg.changeset_pad_hours}h backward pad"
         )
         info(f"Changesets: {len(urls)} files (seq {cs_start}-{cs_end}), {pad_note}.")
+        if len(urls) > 5000:
+            warn(
+                f"Hashtag/changeset filtering downloads the per-minute changeset stream for the live "
+                f"tail ({len(urls):,} files here). This is slow over a busy network and resumes from "
+                f"cache if interrupted; a shorter range or waiting for the dataset to cover more months "
+                f"reduces it."
+            )
 
         cs_frontier_ts = cs_repl.sequence_to_timestamp(cs_end)
 

tests/test_download_resilience.py

@@ -0,0 +1,28 @@
+"""Downloads stay polite and a network failure becomes a clear, resumable error."""
+
+
+import pytest
+import requests
+
+from osmsg import pipeline
+from osmsg.exceptions import OsmsgError
+
+
+def test_download_concurrency_is_capped():
+    assert pipeline._DOWNLOAD_WORKERS <= 4
+
+
+def test_download_all_wraps_network_error(monkeypatch, tmp_path):
+    def boom(url, **kwargs):
+        raise requests.exceptions.ConnectTimeout("planet timed out")
+
+    monkeypatch.setattr(pipeline, "download_osm_file", boom)
+    with pytest.raises(OsmsgError, match="Re-run to resume"):
+        pipeline._download_all(
+            ["https://planet.openstreetmap.org/replication/changesets/007/035/882.osm.gz"],
+            "changeset",
+            8,
+            None,
+            tmp_path,
+            "changesets",
+        )

tests/test_gui.py

@@ -5,7 +5,7 @@
 import pytest
 
 from osmsg.exceptions import OsmsgError
-from osmsg.gui import PRESETS, build_config, preset_range
+from osmsg.gui import ABOUT_LINKS, PRESETS, build_config, preset_range
 
 UTC = dt.UTC
 NOW = dt.datetime(2026, 6, 24, 12, 0, tzinfo=UTC)
@@ -68,3 +68,10 @@ def test_every_preset_resolves():
     for name in PRESETS:
         start, end = preset_range(name, NOW)
         assert start < end
+
+
+def test_about_links():
+    urls = {url for _label, url in ABOUT_LINKS}
+    assert "https://github.com/osgeonepal/osmsg" in urls
+    assert "https://github.com/osgeonepal/osmsg/issues" in urls
+    assert "https://github.com/sponsors/kshitijrajsharma" in urls