feat: add inference service type for AI gateway configuration

Add service_type discriminator and inference config map to services,
enabling provider-aware token limiting, cost attribution, model routing,
guardrails, streaming, and cross-provider fallback in generated KDL.

Author: raffaelschneider

lib/sentinel_cp/config_export.ex

@@ -199,7 +199,8 @@ defmodule SentinelCp.ConfigExport do
         "route_path" => s.route_path,
         "upstream_url" => s.upstream_url,
         "enabled" => s.enabled,
-        "position" => s.position
+        "position" => s.position,
+        "service_type" => s.service_type
       }
 
       # Include non-default config maps
@@ -212,6 +213,7 @@ defmodule SentinelCp.ConfigExport do
       |> maybe_add("retry", s.retry, %{})
       |> maybe_add("compression", s.compression, %{})
       |> maybe_add("security", s.security, %{})
+      |> maybe_add("inference", s.inference, %{})
     end)
   end
 
@@ -291,7 +293,9 @@ defmodule SentinelCp.ConfigExport do
           route_path: svc_data["route_path"],
           upstream_url: svc_data["upstream_url"],
           enabled: Map.get(svc_data, "enabled", true),
-          position: Map.get(svc_data, "position", 0)
+          position: Map.get(svc_data, "position", 0),
+          service_type: Map.get(svc_data, "service_type", "standard"),
+          inference: Map.get(svc_data, "inference", %{})
         }
 
         case SentinelCp.Services.create_service(attrs) do

lib/sentinel_cp/services.ex

@@ -25,6 +25,7 @@ defmodule SentinelCp.Services do
     query =
       Enum.reduce(opts, query, fn
         {:enabled, enabled}, q -> where(q, [s], s.enabled == ^enabled)
+        {:service_type, type}, q -> where(q, [s], s.service_type == ^type)
         _, q -> q
       end)
 
@@ -781,7 +782,8 @@ defmodule SentinelCp.Services do
       status: if(service.enabled, do: "enabled", else: "disabled"),
       metadata: %{
         route_path: service.route_path,
-        upstream_url: service.upstream_url
+        upstream_url: service.upstream_url,
+        service_type: service.service_type
       }
     }
   end

lib/sentinel_cp/services/built_in_templates.ex

@@ -103,6 +103,26 @@ defmodule SentinelCp.Services.BuiltInTemplates do
       },
       is_builtin: true,
       version: 1
+    },
+    %{
+      name: "LLM Inference Gateway",
+      category: "inference",
+      description: "AI inference gateway with token rate limiting, cost tracking, and streaming.",
+      template_data: %{
+        "upstream_url" => "http://inference-backend:8080",
+        "route_path" => "/v1/*",
+        "service_type" => "inference",
+        "timeout_seconds" => 300,
+        "inference" => %{
+          "provider" => "openai",
+          "token_rate_limit" => %{"tokens_per_minute" => 100_000, "burst_allowance" => 1.5},
+          "token_budget" => %{"period" => "monthly", "limit" => 10_000_000, "alert_threshold" => 0.8, "enforcement" => "block"},
+          "cost_attribution" => %{"currency" => "USD", "models" => [%{"pattern" => "*", "input_cost_per_1k" => 0.01, "output_cost_per_1k" => 0.03}]},
+          "streaming" => %{"enabled" => true, "format" => "sse"}
+        }
+      },
+      is_builtin: true,
+      version: 1
     }
   ]
 

lib/sentinel_cp/services/kdl_generator.ex

@@ -62,7 +62,7 @@ defmodule SentinelCp.Services.KdlGenerator do
   defp resolve_service_configs(services, project_id, environment) do
     config_fields = [:headers, :cors, :cache, :retry, :rate_limit, :health_check,
                      :access_control, :compression, :security, :request_transform,
-                     :response_transform, :path_rewrite]
+                     :response_transform, :path_rewrite, :inference]
 
     Enum.reduce_while(services, {:ok, []}, fn service, {:ok, acc} ->
       case resolve_service_config_maps(service, config_fields, project_id, environment) do
@@ -220,6 +220,13 @@ defmodule SentinelCp.Services.KdlGenerator do
   defp build_route(%Service{} = service, group_map, cert_map, auth_policy_map, middleware_chain) do
     lines = ["    route #{inspect(service.route_path)} {"]
 
+    lines =
+      if service.service_type && service.service_type != "standard" do
+        lines ++ ["        service-type #{inspect(service.service_type)}"]
+      else
+        lines
+      end
+
     lines =
       cond do
         service.upstream_url ->
@@ -259,6 +266,7 @@ defmodule SentinelCp.Services.KdlGenerator do
     lines = lines ++ build_request_transform_block(service.request_transform)
     lines = lines ++ build_response_transform_block(service.response_transform)
     lines = lines ++ build_traffic_split_block(service.traffic_split, group_map)
+    lines = lines ++ build_inference_block(service.inference)
 
     # Append middleware chain blocks (after inline fields)
     lines = lines ++ build_middleware_chain(middleware_chain)
@@ -466,6 +474,143 @@ defmodule SentinelCp.Services.KdlGenerator do
     end
   end
 
+  defp build_inference_block(inf) when inf == %{} or inf == nil, do: []
+
+  defp build_inference_block(inf) do
+    lines = ["        inference {"]
+
+    # Provider
+    lines =
+      case Map.get(inf, "provider") do
+        nil -> lines
+        provider -> lines ++ ["            provider #{inspect(provider)}"]
+      end
+
+    # Token rate limit
+    lines = lines ++ build_inference_sub_block(Map.get(inf, "token_rate_limit"), "rate-limit")
+
+    # Token budget
+    lines = lines ++ build_inference_sub_block(Map.get(inf, "token_budget"), "budget")
+
+    # Cost attribution
+    lines = lines ++ build_cost_attribution_block(Map.get(inf, "cost_attribution"))
+
+    # Model routing
+    lines = lines ++ build_model_routing_block(Map.get(inf, "model_routing"))
+
+    # Guardrails
+    lines = lines ++ build_guardrails_block(Map.get(inf, "guardrails"))
+
+    # Fallback
+    lines = lines ++ build_fallback_block(Map.get(inf, "fallback"))
+
+    # Streaming
+    lines = lines ++ build_inference_sub_block(Map.get(inf, "streaming"), "streaming")
+
+    lines ++ ["        }"]
+  end
+
+  defp build_inference_sub_block(nil, _name), do: []
+  defp build_inference_sub_block(map, _name) when map == %{}, do: []
+
+  defp build_inference_sub_block(map, name) do
+    inner =
+      map
+      |> Enum.sort_by(fn {k, _} -> k end)
+      |> Enum.map(fn {key, value} -> "                #{key} #{format_value(value)}" end)
+
+    ["            #{name} {"] ++ inner ++ ["            }"]
+  end
+
+  defp build_cost_attribution_block(nil), do: []
+  defp build_cost_attribution_block(ca) when ca == %{}, do: []
+
+  defp build_cost_attribution_block(ca) do
+    lines = ["            cost-attribution {"]
+
+    lines =
+      case Map.get(ca, "currency") do
+        nil -> lines
+        currency -> lines ++ ["                currency #{inspect(currency)}"]
+      end
+
+    models = Map.get(ca, "models", [])
+
+    model_lines =
+      Enum.flat_map(models, fn model ->
+        pattern = Map.get(model, "pattern", "*")
+        inner =
+          model
+          |> Map.drop(["pattern"])
+          |> Enum.sort_by(fn {k, _} -> k end)
+          |> Enum.map(fn {key, value} -> "                    #{key} #{format_value(value)}" end)
+
+        ["                model #{inspect(pattern)} {"] ++ inner ++ ["                }"]
+      end)
+
+    lines ++ model_lines ++ ["            }"]
+  end
+
+  defp build_model_routing_block(nil), do: []
+  defp build_model_routing_block(mr) when mr == %{}, do: []
+
+  defp build_model_routing_block(mr) do
+    lines = ["            model-routing {"]
+
+    routes = Map.get(mr, "routes", [])
+
+    route_lines =
+      Enum.flat_map(routes, fn route ->
+        pattern = Map.get(route, "pattern", "*")
+        inner =
+          route
+          |> Map.drop(["pattern"])
+          |> Enum.sort_by(fn {k, _} -> k end)
+          |> Enum.map(fn {key, value} -> "                    #{key} #{format_value(value)}" end)
+
+        ["                route #{inspect(pattern)} {"] ++ inner ++ ["                }"]
+      end)
+
+    lines ++ route_lines ++ ["            }"]
+  end
+
+  defp build_guardrails_block(nil), do: []
+  defp build_guardrails_block(gr) when gr == %{}, do: []
+
+  defp build_guardrails_block(gr) do
+    lines = ["            guardrails {"]
+
+    lines = lines ++ build_inference_sub_block(Map.get(gr, "prompt_injection"), "prompt-injection")
+    lines = lines ++ build_inference_sub_block(Map.get(gr, "pii_detection"), "pii-detection")
+
+    lines ++ ["            }"]
+  end
+
+  defp build_fallback_block(nil), do: []
+  defp build_fallback_block(fb) when fb == %{}, do: []
+
+  defp build_fallback_block(fb) do
+    lines = ["            fallback {"]
+
+    mappings = Map.get(fb, "model_mapping", [])
+
+    mapping_lines =
+      Enum.map(mappings, fn mapping ->
+        from = Map.get(mapping, "from", "")
+        to = Map.get(mapping, "to", "")
+        "                model-mapping #{inspect(from)} #{inspect(to)}"
+      end)
+
+    # Add other top-level fallback keys
+    other_lines =
+      fb
+      |> Map.drop(["model_mapping"])
+      |> Enum.sort_by(fn {k, _} -> k end)
+      |> Enum.map(fn {key, value} -> "                #{key} #{format_value(value)}" end)
+
+    lines ++ other_lines ++ mapping_lines ++ ["            }"]
+  end
+
   defp build_middleware_chain([]), do: []
 
   defp build_middleware_chain(service_middlewares) do

lib/sentinel_cp/services/service.ex

@@ -34,6 +34,8 @@ defmodule SentinelCp.Services.Service do
     field :request_transform, :map, default: %{}
     field :response_transform, :map, default: %{}
     field :traffic_split, :map, default: %{}
+    field :service_type, :string, default: "standard"
+    field :inference, :map, default: %{}
     field :redirect_url, :string
 
     belongs_to :project, SentinelCp.Projects.Project
@@ -74,6 +76,8 @@ defmodule SentinelCp.Services.Service do
       :request_transform,
       :response_transform,
       :traffic_split,
+      :service_type,
+      :inference,
       :redirect_url,
       :upstream_group_id,
       :certificate_id,
@@ -86,6 +90,8 @@ defmodule SentinelCp.Services.Service do
     |> validate_length(:name, min: 1, max: 100)
     |> validate_route_path()
     |> validate_route_type()
+    |> validate_inclusion(:service_type, ~w(standard inference))
+    |> validate_inference_config()
     |> generate_slug()
     |> validate_slug()
     |> unique_constraint([:project_id, :slug], error_key: :slug)
@@ -117,6 +123,8 @@ defmodule SentinelCp.Services.Service do
       :request_transform,
       :response_transform,
       :traffic_split,
+      :service_type,
+      :inference,
       :redirect_url,
       :upstream_group_id,
       :certificate_id,
@@ -128,6 +136,34 @@ defmodule SentinelCp.Services.Service do
     |> validate_length(:name, min: 1, max: 100)
     |> validate_route_path()
     |> validate_route_type()
+    |> validate_inclusion(:service_type, ~w(standard inference))
+    |> validate_inference_config()
+  end
+
+  defp validate_inference_config(changeset) do
+    service_type = get_field(changeset, :service_type)
+    inference = get_field(changeset, :inference)
+
+    case service_type do
+      "inference" ->
+        cond do
+          not is_map(inference) or inference == %{} ->
+            add_error(changeset, :inference, "is required when service_type is inference")
+
+          Map.get(inference, "provider") not in ~w(openai anthropic generic) ->
+            add_error(changeset, :inference, "must include a valid provider (openai, anthropic, generic)")
+
+          true ->
+            changeset
+        end
+
+      _ ->
+        if is_map(inference) and inference != %{} do
+          add_error(changeset, :inference, "must be empty for standard services")
+        else
+          changeset
+        end
+    end
   end
 
   defp validate_route_path(changeset) do

lib/sentinel_cp/services/service_template.ex

@@ -11,7 +11,7 @@ defmodule SentinelCp.Services.ServiceTemplate do
   @primary_key {:id, :binary_id, autogenerate: true}
   @foreign_key_type :binary_id
 
-  @categories ~w(api web websocket static auth utility)
+  @categories ~w(api web websocket static auth utility inference)
 
   schema "service_templates" do
     field :name, :string

priv/repo/migrations/20260213140001_add_inference_to_services.exs

@@ -0,0 +1,12 @@
+defmodule SentinelCp.Repo.Migrations.AddInferenceToServices do
+  use Ecto.Migration
+
+  def change do
+    alter table(:services) do
+      add :service_type, :string, default: "standard"
+      add :inference, :map, default: %{}
+    end
+
+    create index(:services, [:project_id, :service_type])
+  end
+end