feat: add blue-green gradual traffic shifting, OTel tracing, and automated rollback

Blue-green deployments now use multi-step traffic shifting (deploy green →
shift traffic through configurable steps → deploy blue) with operator
controls for advancing traffic, swapping slots, and instant rollback.
Adds validation period enforcement, health-gate-based auto-rollback for
all strategies, and extends rollout templates to support all four strategies.
Also integrates OpenTelemetry tracing for observability.

Author: raffaelschneider

config/config.exs

@@ -94,6 +94,9 @@ config :sentinel_cp, :github_webhook,
 # ACME / Let's Encrypt configuration
 config :sentinel_cp, :acme, directory_url: "https://acme-v02.api.letsencrypt.org/directory"
 
+# OpenTelemetry (disabled by default, enable via OTEL_EXPORTER_OTLP_ENDPOINT in runtime.exs)
+config :opentelemetry, span_processor: :batch, traces_exporter: :none
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"

config/runtime.exs

@@ -40,6 +40,12 @@ if s3_endpoint = System.get_env("S3_ENDPOINT") do
     ]
 end
 
+# OpenTelemetry exporter (enable by setting OTEL_EXPORTER_OTLP_ENDPOINT)
+if otlp_endpoint = System.get_env("OTEL_EXPORTER_OTLP_ENDPOINT") do
+  config :opentelemetry_exporter, otlp_protocol: :http_protobuf, otlp_endpoint: otlp_endpoint
+  config :opentelemetry, span_processor: :batch, traces_exporter: {:opentelemetry_exporter, %{}}
+end
+
 # Bundle compiler
 if sentinel_binary = System.get_env("SENTINEL_BINARY") do
   config :sentinel_cp, SentinelCp.Bundles.Compiler, sentinel_binary: sentinel_binary

config/test.exs

@@ -31,6 +31,9 @@ config :sentinel_cp, Oban, testing: :inline
 # Mark test environment (used to skip async notifications)
 config :sentinel_cp, :env, :test
 
+# Disable OpenTelemetry trace export in tests
+config :opentelemetry, traces_exporter: :none
+
 # Initialize plugs at runtime for faster test compilation
 config :phoenix, :plug_init_mode, :runtime
 

lib/sentinel_cp/application.ex

@@ -7,6 +7,10 @@ defmodule SentinelCp.Application do
 
   @impl true
   def start(_type, _args) do
+    # Set up OpenTelemetry auto-instrumentation
+    OpentelemetryPhoenix.setup()
+    OpentelemetryEcto.setup([:sentinel_cp, :repo])
+
     children = [
       SentinelCpWeb.Telemetry,
       SentinelCp.Repo,

lib/sentinel_cp/bundles/compile_worker.ex

@@ -9,12 +9,19 @@ defmodule SentinelCp.Bundles.CompileWorker do
 
   require Logger
 
+  alias SentinelCp.{Audit, Plugins, Services}
   alias SentinelCp.Bundles
   alias SentinelCp.Bundles.{Compiler, Risk, Signing, Storage}
-  alias SentinelCp.{Audit, Plugins, Services}
+  alias SentinelCp.Observability.Tracer
 
   @impl Oban.Worker
   def perform(%Oban.Job{args: %{"bundle_id" => bundle_id}}) do
+    Tracer.trace_compilation(bundle_id, fn ->
+      do_perform(bundle_id)
+    end)
+  end
+
+  defp do_perform(bundle_id) do
     bundle = Bundles.get_bundle!(bundle_id)
 
     Logger.info("Starting compilation for bundle #{bundle_id}")

lib/sentinel_cp/observability/tracer.ex

@@ -1,58 +1,29 @@
 defmodule SentinelCp.Observability.Tracer do
   @moduledoc """
-  Telemetry-based tracing for key control plane operations.
+  OpenTelemetry tracing for key control plane operations.
 
-  Wraps `:telemetry` to emit span events for bundle compilation,
-  rollout lifecycle, webhook processing, and node heartbeats.
-  Compatible with OpenTelemetry exporters when OTEL SDK is configured.
+  Wraps `OpenTelemetry.Tracer.with_span/3` to create spans for bundle
+  compilation, rollout lifecycle, webhook processing, and node heartbeats.
   """
 
-  require Logger
+  require OpenTelemetry.Tracer, as: Tracer
 
   @doc """
-  Executes a function within a traced span.
+  Executes a function within a traced OpenTelemetry span.
 
-  Emits `[:sentinel_cp, span_name, :start]` and `[:sentinel_cp, span_name, :stop]`
-  telemetry events with timing metadata.
+  Attributes from the metadata map are set on the span.
   """
   def span(span_name, metadata, fun) when is_atom(span_name) and is_map(metadata) do
-    start_time = System.monotonic_time()
-    trace_id = generate_trace_id()
-
-    :telemetry.execute(
-      [:sentinel_cp, span_name, :start],
-      %{system_time: System.system_time()},
-      Map.merge(metadata, %{trace_id: trace_id})
-    )
-
-    try do
-      result = fun.()
-
-      duration = System.monotonic_time() - start_time
-
-      :telemetry.execute(
-        [:sentinel_cp, span_name, :stop],
-        %{duration: duration},
-        Map.merge(metadata, %{trace_id: trace_id, result: :ok})
-      )
-
-      result
-    rescue
-      e ->
-        duration = System.monotonic_time() - start_time
-
-        :telemetry.execute(
-          [:sentinel_cp, span_name, :exception],
-          %{duration: duration},
-          Map.merge(metadata, %{
-            trace_id: trace_id,
-            kind: :error,
-            reason: Exception.message(e),
-            stacktrace: __STACKTRACE__
-          })
-        )
-
-        reraise e, __STACKTRACE__
+    attributes = Enum.map(metadata, fn {k, v} -> {to_string(k), to_string(v)} end)
+
+    Tracer.with_span :"sentinel_cp.#{span_name}", %{attributes: attributes} do
+      try do
+        fun.()
+      rescue
+        e ->
+          Tracer.set_status(:error, Exception.message(e))
+          reraise e, __STACKTRACE__
+      end
     end
   end
 
@@ -83,20 +54,4 @@ defmodule SentinelCp.Observability.Tracer do
   def trace_heartbeat(node_id, fun) do
     span(:node_heartbeat, %{node_id: node_id}, fun)
   end
-
-  @doc """
-  Returns all telemetry event prefixes emitted by the tracer.
-  """
-  def event_prefixes do
-    [
-      [:sentinel_cp, :bundle_compilation],
-      [:sentinel_cp, :rollout_tick],
-      [:sentinel_cp, :webhook_processing],
-      [:sentinel_cp, :node_heartbeat]
-    ]
-  end
-
-  defp generate_trace_id do
-    :crypto.strong_rand_bytes(16) |> Base.hex_encode32(case: :lower, padding: false)
-  end
 end