feat: live benchmark ground truth, progress bar, sample size control

- ground_truth.py: from_detections() — stratified sample from detections
  table using rule verdicts as ground truth (confirmed/rule_match = anomaly)
  default 150 events, 35% min anomaly ratio
- comparator.py: progress_cb parameter threaded through multi_run_benchmark
  → BenchmarkComparator → quantum event loop, reports run/event/total
- app.py: Rule-based ground truth as default label source in live mode,
  sample size slider (50-500, default 150), progress bar for both
  single-mode and compare-both benchmark runs

Author: marjatmm-sec

src/benchmark/comparator.py

@@ -10,7 +10,7 @@
   5. Store benchmark_run in DuckDB
   6. Return comparison report
 
-Classical head uses the same 16-dimensional feature vector as the
+Classical head uses the same 24-dimensional feature vector as the
 quantum head — a weighted linear threshold on the feature dimensions
 most predictive of anomalous activity. This ensures apples-to-apples
 comparison: neither head has access to information the other doesn't.
@@ -33,27 +33,7 @@
 
 
 # ── Classical feature-vector classifier ───────────────────────────
-# Weighted linear combination of the 16 feature dimensions.
-# Weights reflect empirical separation between benign and anomaly
-# classes observed in the demo dataset analysis.
-#
-# Dimension index → weight mapping:
-#   0  source_ip_is_private   -0.5  (public IP = more suspicious for scans)
-#   1  source_ip_oct3_norm     0.1
-#   2  source_ip_oct4_norm     0.2
-#   3  source_ip_entropy       0.1
-#   4  severity_norm           1.5  (strong signal)
-#   5  severity_is_high        2.0  (strongest signal)
-#   6  process_risk            1.2
-#   7  process_is_auth         0.8
-#   8  process_is_network      0.8  (raised: kernel DROP = network process)
-#   9  message_has_fail        1.0
-#  10  message_has_auth        0.6
-#  11  message_has_exploit     1.3
-#  12  message_has_scan        1.8  (raised: primary port scan signal)
-#  13  repeat_source           0.9
-#  14  hour_of_day_norm        0.0
-#  15  is_off_hours            0.4
+# Weighted linear combination of the 24 feature dimensions (v2).
 
 _FV_WEIGHTS = [
     # ── v1 base dimensions (0-15) ──────────────────────────────────
@@ -86,8 +66,6 @@
 
 # Decision threshold — optimised for v2 24-dim vector
 # F1=0.989, Acc=99%, FPR=1.8%, 0 FN on 100-event demo dataset
-# One persistent FP: legitimate admin SSH session (benign_auth, score=5.71)
-# — structurally indistinguishable from brute force at feature level
 _FV_THRESHOLD = 4.974
 
 
@@ -102,20 +80,22 @@ def _parse_feature_vector(event: dict) -> list[float]:
     """Extract feature vector from event dict, handling JSON string storage."""
     fv = event.get("feature_vector")
     if fv is None:
-        return [0.0] * 16
+        return [0.0] * 24
     if isinstance(fv, str):
         try:
             fv = json.loads(fv)
         except Exception:
-            return [0.0] * 16
+            try:
+                import ast
+                fv = ast.literal_eval(fv)
+            except Exception:
+                return [0.0] * 24
     if isinstance(fv, list):
         return [float(x) for x in fv]
-    return [0.0] * 16
+    return [0.0] * 24
 
 
-# ── Module-level decoder singleton ───────────────────────────────
-# Shared across all BenchmarkComparator instances in the same process
-# to avoid reloading model weights on every benchmark run.
+# ── Module-level decoder singleton ────────────────────────────────
 _shared_decoder = None
 
 def _get_shared_decoder():
@@ -138,35 +118,27 @@ def __init__(
         scenario_name: str = "",
         noise_enabled: Optional[bool] = None,
         backend:       Optional[str] = None,
+        progress_cb:   object = None,
+        run_idx:       int = 0,
     ):
         self.circuit_type  = circuit_type
         self.scenario_name = scenario_name or f"benchmark_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-        self.q_runner      = QuantumRunner(circuit_type=circuit_type,
-                                            decoder=_get_shared_decoder(),
-                                            noise_enabled=noise_enabled,
-                                            backend=backend)
+        self.progress_cb   = progress_cb
+        self.run_idx       = run_idx
+        self.q_runner      = QuantumRunner(
+            circuit_type=circuit_type,
+            decoder=_get_shared_decoder(),
+            noise_enabled=noise_enabled,
+            backend=backend,
+        )
 
     def run(
         self,
-        events:       list[dict],
-        ground_truth: list[int],
+        events:            list[dict],
+        ground_truth:      list[int],
         classical_results: Optional[list[dict]] = None,
-        notes:        str = "",
+        notes:             str = "",
     ) -> dict:
-        """
-        Execute a full benchmark run.
-
-        Args:
-            events:            List of normalised ECS events
-            ground_truth:      Binary labels (1=anomaly, 0=benign)
-                               Must match length of events
-            classical_results: Pre-computed classical results.
-                               If None, uses feature-vector classifier.
-            notes:             Optional notes stored with the run
-
-        Returns:
-            Full comparison report dict
-        """
         if len(events) != len(ground_truth):
             raise ValueError(
                 f"events ({len(events)}) and ground_truth "
@@ -175,7 +147,7 @@ def run(
         if not events:
             raise ValueError("events list cannot be empty")
 
-        # ── Classical head ────────────────────────────────────────
+        # ── Classical head ─────────────────────────────────────────
         if classical_results:
             c_predictions = [r.get("prediction", 0) for r in classical_results]
             c_latencies   = [r.get("latency_ms", 0.0) for r in classical_results]
@@ -186,82 +158,58 @@ def run(
             c_predictions, ground_truth, c_latencies, label="classical"
         )
 
-        # ── Quantum head ──────────────────────────────────────────
+        # ── Quantum head ───────────────────────────────────────────
         q_predictions = []
         q_latencies   = []
 
-        for event in events:
+        for _ev_idx, event in enumerate(events):
             result = self.q_runner.run(event)
             q_predictions.append(result["prediction"])
             q_latencies.append(result["latency_ms"])
+            if self.progress_cb:
+                try:
+                    self.progress_cb(self.run_idx, _ev_idx, len(events))
+                except Exception:
+                    pass
 
         quantum_metrics = compute_metrics(
             q_predictions, ground_truth, q_latencies, label="quantum"
         )
 
-        # ── Compare ───────────────────────────────────────────────
+        # ── Compare & store ────────────────────────────────────────
         comparison = compare_metrics(classical_metrics, quantum_metrics)
+        run_id = self._store_benchmark(classical_metrics, quantum_metrics, events, notes)
 
-        # ── Store ─────────────────────────────────────────────────
-        run_id = self._store_benchmark(
-            classical_metrics, quantum_metrics,
-            events, notes
-        )
-
-        comparison["run_id"]              = run_id
-        comparison["scenario_name"]       = self.scenario_name
-        comparison["n_events"]            = len(events)
-        comparison["circuit_type"]        = self.circuit_type
-        comparison["created_at"]          = datetime.now(timezone.utc).isoformat()
+        comparison["run_id"]                = run_id
+        comparison["scenario_name"]         = self.scenario_name
+        comparison["n_events"]              = len(events)
+        comparison["circuit_type"]          = self.circuit_type
+        comparison["created_at"]            = datetime.now(timezone.utc).isoformat()
         comparison["classical_predictions"] = c_predictions
         comparison["quantum_predictions"]   = q_predictions
 
         return comparison
 
-    # ── Classifiers ───────────────────────────────────────────────
-
-    def _fv_classifier(
-        self,
-        events: list[dict],
-    ) -> tuple[list[int], list[float]]:
-        """
-        Feature-vector weighted linear classifier.
-        Uses the same 16-dimensional feature vector as the quantum head.
-        Predicts anomaly (1) if weighted score >= _FV_THRESHOLD.
-        Returns (predictions, latencies_ms).
-        """
+    def _fv_classifier(self, events: list[dict]) -> tuple[list[int], list[float]]:
         import time
-        predictions = []
-        latencies   = []
-
+        predictions, latencies = [], []
         for event in events:
             start = time.perf_counter()
             fv    = _parse_feature_vector(event)
             score = _fv_score(fv)
             predictions.append(1 if score >= _FV_THRESHOLD else 0)
             latencies.append((time.perf_counter() - start) * 1000)
-
         return predictions, latencies
 
-    def _severity_classifier(
-        self,
-        events: list[dict],
-    ) -> tuple[list[int], list[float]]:
-        """
-        Legacy severity-threshold classifier. Kept for reference.
-        Predicts anomaly (1) if event_severity >= 50.
-        NOT used in benchmark — replaced by _fv_classifier.
-        """
+    def _severity_classifier(self, events: list[dict]) -> tuple[list[int], list[float]]:
+        """Legacy severity-threshold classifier — not used in benchmark."""
         import time
-        predictions = []
-        latencies   = []
-
+        predictions, latencies = [], []
         for event in events:
             start = time.perf_counter()
             sev   = event.get("event_severity") or 0
             predictions.append(1 if sev >= 50 else 0)
             latencies.append((time.perf_counter() - start) * 1000)
-
         return predictions, latencies
 
     def _store_benchmark(
@@ -271,10 +219,8 @@ def _store_benchmark(
         events:    list[dict],
         notes:     str,
     ) -> str:
-        """Store benchmark run in DuckDB. Returns run ID."""
         run_id    = str(uuid.uuid4())
         event_ids = [e.get("id", "") for e in events]
-
         con = get_connection()
         con.execute("""
             INSERT OR REPLACE INTO benchmark_runs (
@@ -287,16 +233,10 @@ def _store_benchmark(
                 notes
             ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         """, [
-            run_id,
-            datetime.now(timezone.utc),
-            self.scenario_name,
-            event_ids,
-            classical.accuracy, classical.f1,
-            classical.fpr,      classical.avg_latency_ms,
-            quantum.accuracy,   quantum.f1,
-            quantum.fpr,        quantum.avg_latency_ms,
-            classical.efficacy, quantum.efficacy,
-            notes,
+            run_id, datetime.now(timezone.utc), self.scenario_name, event_ids,
+            classical.accuracy, classical.f1, classical.fpr, classical.avg_latency_ms,
+            quantum.accuracy,   quantum.f1,   quantum.fpr,   quantum.avg_latency_ms,
+            classical.efficacy, quantum.efficacy, notes,
         ])
         con.close()
         return run_id
@@ -309,11 +249,10 @@ def quick_benchmark(
     scenario:     str = "",
 ) -> dict:
     """Convenience function for a one-line benchmark run."""
-    comparator = BenchmarkComparator(
+    return BenchmarkComparator(
         circuit_type=circuit_type,
         scenario_name=scenario,
-    )
-    return comparator.run(events, ground_truth)
+    ).run(events, ground_truth)
 
 
 def multi_run_benchmark(
@@ -325,11 +264,11 @@ def multi_run_benchmark(
     classical_results: list[dict] | None = None,
     noise_enabled:     bool | None = None,
     backend:           str | None = None,
+    progress_cb:       object = None,
 ) -> dict:
     """
-    Run benchmark N times and return averaged results with
-    standard deviation — giving statistically meaningful results
-    at low qubit counts where single-run variance is high.
+    Run benchmark N times and return averaged results with std dev.
+    progress_cb(run_idx, event_idx, total_events) called per event.
     """
     if n_runs < 1:
         raise ValueError("n_runs must be >= 1")
@@ -341,6 +280,8 @@ def multi_run_benchmark(
             scenario_name=f"{scenario}_run{i+1}" if scenario else f"run{i+1}",
             noise_enabled=noise_enabled,
             backend=backend,
+            progress_cb=progress_cb,
+            run_idx=i,
         )
         report = comp.run(events, ground_truth, classical_results=classical_results)
         runs.append(report)
@@ -356,7 +297,6 @@ def agg(key: str, subkey: str) -> dict:
             "max":  round(max(vals), 4),
         }
 
-    # Aggregate predictions across runs (majority vote per event)
     def majority_preds(key: str) -> list[int]:
         all_preds = [r.get(key, []) for r in runs]
         if not all_preds or not all_preds[0]:
@@ -369,10 +309,10 @@ def majority_preds(key: str) -> list[int]:
         return result
 
     return {
-        "n_runs":                   n_runs,
-        "circuit_type":             circuit_type,
-        "scenario":                 scenario,
-        "n_events":                 len(events),
+        "n_runs":       n_runs,
+        "circuit_type": circuit_type,
+        "scenario":     scenario,
+        "n_events":     len(events),
         "classical": {
             "efficacy":       agg("classical", "efficacy"),
             "accuracy":       agg("classical", "accuracy"),
@@ -400,7 +340,6 @@ def majority_preds(key: str) -> list[int]:
 
 
 def _determine_overall_winner(runs: list[dict]) -> str:
-    """Determine winner across multiple runs by average efficacy delta."""
     import statistics
     deltas = [r["delta"]["efficacy"] for r in runs]
     avg_delta = statistics.mean(deltas)