Recover detection runs interrupted by a restart

A detection run completes inside its request, so one still marked active at
startup did not finish. Leaving it that way would have been worse than
untidy: an overlapping run is refused while one looks active, so a run left
behind by a crash would have blocked that Dockyard's detection permanently.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: chriswayneh

backend/app/detection/runner.py

@@ -123,6 +123,25 @@ def get_run(session: Session, dockyard_id: int, run_id: int) -> DetectionRun | N
     )
 
 
+def recover_interrupted_runs(session: Session) -> int:
+    """Mark detection runs that a restart interrupted.
+
+    A detection run completes inside its request, so one still marked active at
+    startup did not finish. Saying so matters twice over: the record is honest,
+    and an overlapping run is refused while one looks active, so a run left
+    behind by a crash would otherwise block this Dockyard's detection for good.
+    """
+    interrupted = list(
+        session.scalars(select(DetectionRun).where(DetectionRun.status.in_(ACTIVE_STATUSES)))
+    )
+    for run in interrupted:
+        run.status = str(DetectionRunStatus.FAILED)
+        run.error = "Interrupted by a RedDock restart"
+        run.completed_at = datetime.now(UTC)
+    session.commit()
+    return len(interrupted)
+
+
 def execute_run(session: Session, run: DetectionRun) -> DetectionRun:
     """Run every registered detector over the Dockyard snapshot. Never raises."""
     run.status = str(DetectionRunStatus.RUNNING)

backend/app/main.py

@@ -9,6 +9,7 @@
 from app.api import router
 from app.config import get_settings
 from app.database import SessionLocal, initialize_database
+from app.detection.runner import recover_interrupted_runs as recover_interrupted_detections
 from app.discovery.runner import recover_interrupted_runs
 
 STATIC_DIRECTORY = Path(__file__).resolve().parents[2] / "static"
@@ -21,10 +22,14 @@ async def lifespan(_: FastAPI):
     initialize_database()
     with SessionLocal() as session:
         # A run that was in flight when the process stopped did not finish.
-        # Saying so is more useful than leaving it looking active forever.
+        # Saying so is more useful than leaving it looking active forever, and a
+        # detection run left active would keep refusing the next one.
         interrupted = recover_interrupted_runs(session)
+        detections = recover_interrupted_detections(session)
     if interrupted:
         logger.warning("Marked %s discovery run(s) as interrupted by restart", interrupted)
+    if detections:
+        logger.warning("Marked %s detection run(s) as interrupted by restart", detections)
     yield
 
 

backend/tests/test_detection_runner.py

@@ -672,3 +672,22 @@ def test_detection_uses_the_snapshot_time_for_seen_timestamps(
     stored = finding.first_seen.replace(tzinfo=UTC)
     assert finding.first_seen == finding.last_seen
     assert stored >= before.replace(microsecond=0)
+
+
+def test_a_detection_run_interrupted_by_a_restart_is_marked_and_unblocks_the_next(
+    endpoint: Recorder, monkeypatch: pytest.MonkeyPatch
+):
+    """An overlapping run is refused, so a stale active run must not be permanent."""
+    from app.models import DetectionRun
+
+    stale = DetectionRun(dockyard_id=endpoint.dockyard_id, status="running")
+    endpoint.session.add(stale)
+    endpoint.session.commit()
+
+    assert detection_runner.recover_interrupted_runs(endpoint.session) == 1
+    endpoint.session.refresh(stale)
+    assert stale.status == "failed"
+    assert "restart" in stale.error
+
+    install(monkeypatch, StubDetector())
+    assert detect(endpoint).status == "completed"