feat: add SLO/SLI dashboard, alerting UI, and periodic SLI computation worker

Wire up the existing Observability backend (SLOs, alert rules, alert states)
with a full LiveView UI layer, routing, sidebar navigation, and a periodic
SLI recomputation Oban worker. Adds PromEx metrics and dashboard integration.

Author: raffaelschneider

lib/sentinel_cp/dashboard.ex

@@ -37,12 +37,16 @@ defmodule SentinelCp.Dashboard do
   Returns overview for a single project.
   """
   def get_project_overview(project_id) do
+    alias SentinelCp.Observability
+
     %{
       node_stats: get_project_node_stats(project_id),
       active_rollouts: count_active_rollouts([project_id]),
       recent_bundles: count_recent_bundles([project_id], 7),
       latest_bundles: list_latest_bundles(project_id, 5),
-      latest_rollouts: list_latest_rollouts(project_id, 5)
+      latest_rollouts: list_latest_rollouts(project_id, 5),
+      slo_summary: Observability.slo_summary(project_id),
+      firing_alert_count: Observability.firing_alert_count(project_id)
     }
   end
 

lib/sentinel_cp/observability.ex

@@ -31,6 +31,9 @@ defmodule SentinelCp.Observability do
   @doc "Gets an SLO by ID."
   def get_slo(id), do: Repo.get(Slo, id)
 
+  @doc "Gets an SLO by ID, raising if not found."
+  def get_slo!(id), do: Repo.get!(Slo, id)
+
   @doc "Lists all SLOs for a project."
   def list_slos(project_id) do
     from(s in Slo, where: s.project_id == ^project_id, order_by: [asc: s.name])
@@ -43,6 +46,31 @@ defmodule SentinelCp.Observability do
     |> Repo.all()
   end
 
+  @doc "Lists all enabled SLOs across all projects (for the SLI worker)."
+  def list_all_enabled_slos do
+    from(s in Slo, where: s.enabled == true, order_by: [asc: s.inserted_at])
+    |> Repo.all()
+  end
+
+  @doc "Returns a summary of SLO statuses for a project."
+  def slo_summary(project_id) do
+    slos = list_slos(project_id)
+
+    Enum.reduce(slos, %{total: 0, healthy: 0, warning: 0, breached: 0}, fn slo, acc ->
+      status = slo_status(slo)
+
+      acc
+      |> Map.update!(:total, &(&1 + 1))
+      |> Map.update!(status, &(&1 + 1))
+    end)
+  end
+
+  @doc "Returns the status atom for an SLO based on error budget remaining."
+  def slo_status(%Slo{error_budget_remaining: nil}), do: :healthy
+  def slo_status(%Slo{error_budget_remaining: budget}) when budget >= 50.0, do: :healthy
+  def slo_status(%Slo{error_budget_remaining: budget}) when budget > 0.0, do: :warning
+  def slo_status(%Slo{}), do: :breached
+
   @doc "Deletes an SLO."
   def delete_slo(slo), do: Repo.delete(slo)
 
@@ -75,6 +103,9 @@ defmodule SentinelCp.Observability do
   @doc "Gets an alert rule by ID."
   def get_alert_rule(id), do: Repo.get(AlertRule, id)
 
+  @doc "Gets an alert rule by ID, raising if not found."
+  def get_alert_rule!(id), do: Repo.get!(AlertRule, id)
+
   @doc "Lists alert rules for a project."
   def list_alert_rules(project_id) do
     from(r in AlertRule, where: r.project_id == ^project_id, order_by: [asc: r.name])
@@ -140,4 +171,31 @@ defmodule SentinelCp.Observability do
     )
     |> Repo.aggregate(:count)
   end
+
+  @doc "Gets an alert state by ID, raising if not found."
+  def get_alert_state!(id), do: Repo.get!(AlertState, id)
+
+  @doc "Lists firing and pending alert states for a project, with preloaded rules."
+  def list_firing_alerts(project_id) do
+    from(s in AlertState,
+      join: r in AlertRule,
+      on: s.alert_rule_id == r.id,
+      where: r.project_id == ^project_id and s.state in ["firing", "pending"],
+      order_by: [desc: s.started_at],
+      preload: [alert_rule: r]
+    )
+    |> Repo.all()
+  end
+
+  @doc "Lists recent alert states for a rule (paginated history)."
+  def list_recent_alert_states(alert_rule_id, opts \\ []) do
+    limit = Keyword.get(opts, :limit, 50)
+
+    from(s in AlertState,
+      where: s.alert_rule_id == ^alert_rule_id,
+      order_by: [desc: s.started_at],
+      limit: ^limit
+    )
+    |> Repo.all()
+  end
 end

lib/sentinel_cp/observability/sli_worker.ex

@@ -0,0 +1,61 @@
+defmodule SentinelCp.Observability.SliWorker do
+  @moduledoc """
+  Oban worker that periodically recomputes SLI values for all enabled SLOs.
+
+  Runs every 5 minutes, fetches all enabled SLOs across all projects,
+  and calls `SliComputer.compute/1` for each.
+  """
+  use Oban.Worker,
+    queue: :maintenance,
+    max_attempts: 1,
+    unique: [period: 300]
+
+  require Logger
+
+  alias SentinelCp.Observability
+  alias SentinelCp.Observability.SliComputer
+
+  @check_interval_seconds 300
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    slos = Observability.list_all_enabled_slos()
+
+    results =
+      Enum.map(slos, fn slo ->
+        case SliComputer.compute(slo) do
+          {:ok, updated} ->
+            {:ok, updated}
+
+          {:error, reason} ->
+            Logger.warning("SliWorker: failed to compute SLO #{slo.id}: #{inspect(reason)}")
+            {:error, reason}
+        end
+      end)
+
+    ok_count = Enum.count(results, &match?({:ok, _}, &1))
+    Logger.debug("SliWorker: computed #{ok_count}/#{length(slos)} SLOs")
+
+    schedule_next()
+    :ok
+  end
+
+  @doc """
+  Ensures the SLI worker is scheduled. Safe to call multiple times.
+  """
+  def ensure_started do
+    oban_config = Application.get_env(:sentinel_cp, Oban, [])
+
+    unless oban_config[:testing] do
+      %{} |> __MODULE__.new(schedule_in: 120) |> Oban.insert()
+    end
+  end
+
+  defp schedule_next do
+    oban_config = Application.get_env(:sentinel_cp, Oban, [])
+
+    unless oban_config[:testing] do
+      %{} |> __MODULE__.new(schedule_in: @check_interval_seconds) |> Oban.insert()
+    end
+  end
+end

lib/sentinel_cp/prom_ex/sentinel_plugin.ex

@@ -46,6 +46,19 @@ defmodule SentinelCp.PromEx.SentinelPlugin do
           event_name: [:sentinel_cp, :drift, :nodes, :drifted_count],
           description: "Number of currently drifted nodes",
           measurement: :count
+        ),
+        last_value(
+          [:sentinel_cp, :slos, :total],
+          event_name: [:sentinel_cp, :slos, :status_count],
+          description: "Total number of SLOs by status",
+          measurement: :count,
+          tags: [:status]
+        ),
+        last_value(
+          [:sentinel_cp, :alerts, :firing],
+          event_name: [:sentinel_cp, :alerts, :firing_count],
+          description: "Number of currently firing alerts",
+          measurement: :count
         )
       ]
     )
@@ -120,6 +133,26 @@ defmodule SentinelCp.PromEx.SentinelPlugin do
       %{count: drifted_nodes},
       %{}
     )
+
+    # SLO status counts
+    for status <- ["healthy", "warning", "breached"] do
+      count = poll_slo_count(status)
+
+      :telemetry.execute(
+        [:sentinel_cp, :slos, :status_count],
+        %{count: count},
+        %{status: status}
+      )
+    end
+
+    # Firing alerts
+    firing_alerts = poll_firing_alerts()
+
+    :telemetry.execute(
+      [:sentinel_cp, :alerts, :firing_count],
+      %{count: firing_alerts},
+      %{}
+    )
   end
 
   defp poll_node_count(status) do
@@ -170,6 +203,40 @@ defmodule SentinelCp.PromEx.SentinelPlugin do
     _ -> 0
   end
 
+  defp poll_slo_count(status) do
+    import Ecto.Query
+
+    budget_filter =
+      case status do
+        "healthy" ->
+          dynamic([s], is_nil(s.error_budget_remaining) or s.error_budget_remaining >= 50.0)
+
+        "warning" ->
+          dynamic([s], s.error_budget_remaining < 50.0 and s.error_budget_remaining > 0.0)
+
+        "breached" ->
+          dynamic([s], s.error_budget_remaining <= 0.0)
+      end
+
+    SentinelCp.Repo.aggregate(
+      from(s in "slos", where: ^budget_filter),
+      :count
+    )
+  rescue
+    _ -> 0
+  end
+
+  defp poll_firing_alerts do
+    import Ecto.Query
+
+    SentinelCp.Repo.aggregate(
+      from(s in "alert_states", where: s.state == "firing"),
+      :count
+    )
+  rescue
+    _ -> 0
+  end
+
   @doc """
   Emits a telemetry event for drift detection.
   Call this when a drift event is created.

lib/sentinel_cp_web/components/layouts.ex

@@ -190,6 +190,20 @@ defmodule SentinelCpWeb.Layouts do
           current={@path}
           match="/analytics"
         />
+        <.sidebar_link
+          path={~p"/orgs/#{@org_slug}/projects/#{@project_slug}/slos"}
+          icon="hero-chart-pie"
+          label="SLOs"
+          current={@path}
+          match="/slos"
+        />
+        <.sidebar_link
+          path={~p"/orgs/#{@org_slug}/projects/#{@project_slug}/alerts"}
+          icon="hero-bell-alert"
+          label="Alerts"
+          current={@path}
+          match="/alerts"
+        />
 
         <div class="sidebar-section-title mt-4">Settings</div>
         <.sidebar_link

lib/sentinel_cp_web/live/alerts_live/index.ex

@@ -0,0 +1,119 @@
+defmodule SentinelCpWeb.AlertsLive.Index do
+  use SentinelCpWeb, :live_view
+
+  import SentinelCpWeb.SlosLive.Helpers
+
+  alias SentinelCp.{Observability, Projects}
+
+  @refresh_interval 15_000
+
+  @impl true
+  def mount(%{"project_slug" => slug} = params, _session, socket) do
+    org = resolve_org(params)
+
+    case Projects.get_project_by_slug(slug) do
+      nil ->
+        {:ok, push_navigate(socket, to: ~p"/orgs")}
+
+      project ->
+        if connected?(socket), do: :timer.send_interval(@refresh_interval, :refresh)
+        alerts = Observability.list_firing_alerts(project.id)
+
+        {:ok,
+         assign(socket,
+           page_title: "Active Alerts - #{project.name}",
+           org: org,
+           project: project,
+           alerts: alerts
+         )}
+    end
+  end
+
+  @impl true
+  def handle_info(:refresh, socket) do
+    alerts = Observability.list_firing_alerts(socket.assigns.project.id)
+    {:noreply, assign(socket, :alerts, alerts)}
+  end
+
+  @impl true
+  def handle_event("acknowledge", %{"id" => id}, socket) do
+    alert_state = Observability.get_alert_state!(id)
+    user_id = socket.assigns.current_user.id
+    {:ok, _} = Observability.acknowledge_alert(alert_state, user_id)
+    alerts = Observability.list_firing_alerts(socket.assigns.project.id)
+
+    {:noreply,
+     socket
+     |> assign(:alerts, alerts)
+     |> put_flash(:info, "Alert acknowledged.")}
+  end
+
+  @impl true
+  def render(assigns) do
+    ~H"""
+    <div class="space-y-4">
+      <div class="flex items-center justify-between">
+        <h1 class="text-xl font-bold">Alerts</h1>
+        <.link navigate={new_alert_rule_path(@org, @project)} class="btn btn-primary btn-sm">
+          New Rule
+        </.link>
+      </div>
+
+      <.alert_tabs org={@org} project={@project} active="active" />
+
+      <div :if={@alerts == []} class="text-center py-12 text-base-content/50">
+        <.icon name="hero-check-circle" class="size-12 mx-auto mb-2 opacity-50" />
+        <p>No active alerts. All systems normal.</p>
+      </div>
+
+      <table :if={@alerts != []} class="table table-sm">
+        <thead>
+          <tr>
+            <th class="text-xs">State</th>
+            <th class="text-xs">Rule</th>
+            <th class="text-xs">Severity</th>
+            <th class="text-xs">Value</th>
+            <th class="text-xs">Since</th>
+            <th class="text-xs">Actions</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr :for={alert <- @alerts}>
+            <td><.alert_state_badge state={alert.state} /></td>
+            <td>
+              <.link
+                navigate={alert_rule_path(@org, @project, alert.alert_rule)}
+                class="link font-medium"
+              >
+                {alert.alert_rule.name}
+              </.link>
+            </td>
+            <td><.severity_badge severity={alert.alert_rule.severity} /></td>
+            <td class="font-mono text-sm">{format_value(alert.value)}</td>
+            <td class="text-sm">
+              {if alert.started_at,
+                do: Calendar.strftime(alert.started_at, "%Y-%m-%d %H:%M:%S"),
+                else: "—"}
+            </td>
+            <td>
+              <button
+                :if={is_nil(alert.acknowledged_by)}
+                phx-click="acknowledge"
+                phx-value-id={alert.id}
+                class="btn btn-outline btn-xs"
+              >
+                Acknowledge
+              </button>
+              <span :if={alert.acknowledged_by} class="text-xs text-success">Acknowledged</span>
+            </td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    """
+  end
+
+  defp format_value(nil), do: "—"
+  defp format_value(val) when is_float(val), do: Float.round(val, 2) |> to_string()
+  defp format_value(val), do: to_string(val)
+end