fix(health): add health checkup for external check of osmsg

Author: kshitijrajsharma

api/app.py

@@ -1,5 +1,6 @@
 import os
 from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 import asyncpg
@@ -19,6 +20,7 @@
 from .schemas import HealthResponse
 
 FRONTEND_DIST = os.getenv("FRONTEND_DIST")
+_MAX_STALENESS = timedelta(seconds=int(os.getenv("OSMSG_HEALTH_MAX_STALENESS_SECONDS", "1800")))
 DEFAULT_CORS_ORIGINS = (
     "http://localhost:5173",
     "http://127.0.0.1:5173",
@@ -91,6 +93,13 @@ async def health() -> HealthResponse:
         state = await fetch_state()
     except (OSError, asyncpg.PostgresError) as exc:
         raise HTTPException(status_code=503, detail="database unavailable") from exc
+    if state and state["last_ts"] is not None:
+        last_ts = state["last_ts"]
+        if last_ts.tzinfo is None:
+            last_ts = last_ts.replace(tzinfo=UTC)
+        age = datetime.now(UTC) - last_ts
+        if age > _MAX_STALENESS:
+            raise HTTPException(status_code=503, detail=f"stale: newest data is {age} old")
     return HealthResponse(
         status="ok",
         last_seq=state["last_seq"] if state else None,

infra/.env.example

@@ -17,6 +17,12 @@ GA_MEASUREMENT_ID=
 # ticks, each committing its slice, instead of accumulating the whole gap in one pass. Unset = off.
 OSMSG_MAX_UPDATE_WINDOW_HOURS=24
 
+# Kill a tick that overruns this many seconds so a hung run cannot wedge the scheduler (default 1200).
+# OSMSG_TICK_TIMEOUT_SECONDS=1200
+# /health returns 503 when the newest data is older than this many seconds, so an HTTP monitor alerts on a
+# stalled worker (default 1800).
+# OSMSG_HEALTH_MAX_STALENESS_SECONDS=1800
+
 # Postgres DSN for the monthly prune (drops rows now covered by published history). Same db as the
 # worker; the prune runs inside the compose network.
 # OSMSG_PSQL_DSN=postgresql://osmsg:osmsg@db:5432/osmsg

infra/docker-compose.yml

@@ -60,6 +60,7 @@ services:
       OSMSG_BOOTSTRAP: ${OSMSG_BOOTSTRAP:-hour}
       OSMSG_BOOTSTRAP_DAYS: ${OSMSG_BOOTSTRAP_DAYS:-}
       OSMSG_MAX_UPDATE_WINDOW_HOURS: ${OSMSG_MAX_UPDATE_WINDOW_HOURS:-}
+      OSMSG_TICK_TIMEOUT_SECONDS: ${OSMSG_TICK_TIMEOUT_SECONDS:-}
       OSM_USERNAME: ${OSM_USERNAME:-}
       OSM_PASSWORD: ${OSM_PASSWORD:-}
       OSMSG_EXTRA_ARGS: ${OSMSG_EXTRA_ARGS:-}

tests/test_api.py

@@ -142,6 +142,39 @@ async def failing_state():
     assert response.status_code == 503
 
 
+def test_health_reports_503_when_data_is_stale(monkeypatch):
+    from datetime import UTC, datetime, timedelta
+
+    app_module = import_module("api.app")
+
+    async def old_state():
+        now = datetime.now(UTC)
+        return {"last_seq": 1, "last_ts": now - timedelta(hours=5), "updated_at": now}
+
+    monkeypatch.setattr(app_module, "fetch_state", old_state)
+    with TestClient(Litestar(route_handlers=[health])) as client:
+        response = client.get("/health")
+
+    assert response.status_code == 503
+
+
+def test_health_ok_when_data_is_fresh(monkeypatch):
+    from datetime import UTC, datetime
+
+    app_module = import_module("api.app")
+
+    async def fresh_state():
+        now = datetime.now(UTC)
+        return {"last_seq": 1, "last_ts": now, "updated_at": now}
+
+    monkeypatch.setattr(app_module, "fetch_state", fresh_state)
+    with TestClient(Litestar(route_handlers=[health])) as client:
+        response = client.get("/health")
+
+    assert response.status_code == 200
+    assert response.json()["status"] == "ok"
+
+
 def test_window_normalizes_naive_datetime_to_utc():
     from datetime import UTC, datetime