feat(web): configure judge assertions

README.md

@@ -81,6 +81,8 @@ Use a rubric judge when correctness depends on meaning instead of an exact strin
 
 Replace the placeholder with the ID of a configured provider.
 
+In the dashboard, add or edit a suite test case, choose **rubric judge** in the visual assertion editor, then select a built-in judge or enter a custom rubric. The same controls set the judge provider, pass threshold, optional reference answer, and optional grounding context. JSON assertions remain available for direct editing, and persisted judge assertions run unchanged from the dashboard, `mix aludel.eval`, ExUnit, and the Elixir API.
+
 The catalog includes correctness, relevance, faithfulness, safety, refusal quality, PII protection, and hallucination checks. Aludel records the resolved rubric and template version alongside score, reasoning, duration, provider, model, token usage, cost, and structured evaluator status.
 
 See the [evaluation guide](https://hexdocs.pm/aludel/evaluations.html#custom-rubric-judge), [rubric judge guide](https://github.com/ccarvalho-eng/aludel/wiki/Rubric-Judges), and [judge catalog](https://github.com/ccarvalho-eng/aludel/wiki/Judge-Catalog).

guides/evaluations.md

@@ -129,7 +129,7 @@ Use a rubric judge when correctness depends on meaning rather than an exact stri
 ]
 ```
 
-Author rubric assertions in the JSON assertion editor. Scores range from 0 to 100, and Aludel derives the pass or fail result from `threshold`; a model-provided verdict is never trusted. Evaluation evidence is bounded and sent as untrusted JSON data so content under test cannot replace the rubric or output contract.
+In the dashboard's visual assertion editor, choose `rubric judge`, select **Custom rubric**, and configure the rubric, judge provider, threshold, optional reference answer, and optional grounding context. The JSON assertion editor accepts the same fields. Scores range from 0 to 100, and Aludel derives the pass or fail result from `threshold`; a model-provided verdict is never trusted. Evaluation evidence is bounded and sent as untrusted JSON data so content under test cannot replace the rubric or output contract.
 
 For common checks, replace `rubric` with a versioned built-in `template`:
 
@@ -156,6 +156,8 @@ For common checks, replace `rubric` with a versioned built-in `template`:
 
 Templates and custom rubrics are mutually exclusive. Aludel records the resolved rubric and template version with each result, so a historical run retains the criteria it used.
 
+To configure a template visually, choose `rubric judge`, keep **Built-in judge** selected, choose one of the seven versioned templates, and select a configured judge provider. Persisted judge assertions have the same execution behavior in the dashboard, `mix aludel.eval`, ExUnit, file-based suites, and the Elixir API; the visual controls are a dashboard authoring feature.
+
 ### Inspect metric context and evaluator details
 
 Suite execution gives every metric a normalized `Aludel.Evals.Metric.Context` containing the generated output, rendered input, prompt template, variables, messages, documents, metadata, provider, prompt version, and execution details. Expected references remain in assertion configuration. Direct callers can also set `expected` on the context.

guides/features.md

@@ -79,6 +79,8 @@ The suite UI supports:
 
 Built-in metrics are `contains`, `not_contains`, `regex`, `exact_match`, `json_field`, `json_deep_compare`, and `rubric_judge`. See the [Evaluation Guide](evaluations.md) for examples and programmatic policy configuration.
 
+The visual assertion editor configures either a built-in judge template or a custom rubric, a separate judge provider, a 0–100 pass threshold, an optional reference answer, and optional grounding context. The raw JSON editor exposes the same assertion contract. Once saved, judge assertions run through the dashboard, Mix CLI, ExUnit, file-based suites, and the Elixir API.
+
 ## Reusable datasets
 
 Datasets are ordered collections of evaluation examples that can be reused by multiple suites. A dataset entry can contain:

lib/aludel/evals/assertion_parser.ex

@@ -104,6 +104,17 @@ defmodule Aludel.Evals.AssertionParser do
     |> Map.put("assertion_threshold_#{idx}", format_threshold(assertion["threshold"]))
   end
 
+  defp maybe_put_assertion_value(params, idx, %{"type" => "rubric_judge"} = assertion) do
+    params
+    |> Map.put("assertion_rubric_source_#{idx}", rubric_source(assertion))
+    |> Map.put("assertion_template_#{idx}", assertion["template"] || "")
+    |> Map.put("assertion_rubric_#{idx}", assertion["rubric"] || "")
+    |> Map.put("assertion_provider_id_#{idx}", assertion["provider_id"] || "")
+    |> Map.put("assertion_threshold_#{idx}", format_threshold(assertion["threshold"]))
+    |> put_evidence_form_value(idx, "expected", assertion["expected"])
+    |> put_evidence_form_value(idx, "context", assertion["context"])
+  end
+
   defp maybe_put_assertion_value(params, idx, assertion) do
     Map.put(params, "assertion_value_#{idx}", assertion["value"] || "")
   end
@@ -131,12 +142,19 @@ defmodule Aludel.Evals.AssertionParser do
                  idx
                ),
              {:ok, threshold} <-
-               parse_threshold(Map.get(assertion_params, "assertion_threshold_#{idx}", ""), idx) do
+               parse_threshold(
+                 Map.get(assertion_params, "assertion_threshold_#{idx}", ""),
+                 idx,
+                 "json_deep_compare"
+               ) do
           {:ok,
            %{"type" => type, "expected" => expected}
            |> maybe_put_threshold(threshold)}
         end
 
+      "rubric_judge" ->
+        build_visual_rubric_judge(assertion_params, idx, :strict)
+
       _other ->
         {:ok,
          %{
@@ -172,6 +190,9 @@ defmodule Aludel.Evals.AssertionParser do
            Map.get(assertion_params, "assertion_threshold_#{idx}", "")
          )}
 
+      "rubric_judge" ->
+        build_visual_rubric_judge(assertion_params, idx, :preview)
+
       _other ->
         {:ok,
          %{
@@ -363,22 +384,83 @@ defmodule Aludel.Evals.AssertionParser do
      "Assertion at index #{idx}: json_deep_compare type requires valid JSON in the expected payload"}
   end
 
-  defp parse_threshold("", _idx), do: {:ok, nil}
+  defp build_visual_rubric_judge(assertion_params, idx, :strict) do
+    with {:ok, source} <- parse_rubric_source(assertion_params, idx),
+         {:ok, threshold} <-
+           parse_threshold(
+             Map.get(assertion_params, "assertion_threshold_#{idx}", ""),
+             idx,
+             "rubric_judge"
+           ) do
+      assertion =
+        %{
+          "type" => "rubric_judge",
+          "provider_id" => Map.get(assertion_params, "assertion_provider_id_#{idx}", "")
+        }
+        |> Map.merge(source)
+        |> maybe_put_threshold(threshold)
+        |> maybe_put_evidence(assertion_params, idx, "expected")
+        |> maybe_put_evidence(assertion_params, idx, "context")
+
+      {:ok, assertion}
+    end
+  end
 
-  defp parse_threshold(value, idx) when is_binary(value) do
+  defp build_visual_rubric_judge(assertion_params, idx, :preview) do
+    source = preview_rubric_source(assertion_params, idx)
+
+    assertion =
+      %{
+        "type" => "rubric_judge",
+        "provider_id" => Map.get(assertion_params, "assertion_provider_id_#{idx}", "")
+      }
+      |> Map.merge(source)
+      |> maybe_put_preview_threshold(Map.get(assertion_params, "assertion_threshold_#{idx}", ""))
+      |> maybe_put_evidence(assertion_params, idx, "expected")
+      |> maybe_put_evidence(assertion_params, idx, "context")
+
+    {:ok, assertion}
+  end
+
+  defp parse_rubric_source(assertion_params, idx) do
+    case Map.get(assertion_params, "assertion_rubric_source_#{idx}", "template") do
+      "template" ->
+        {:ok, %{"template" => Map.get(assertion_params, "assertion_template_#{idx}", "")}}
+
+      "custom" ->
+        {:ok, %{"rubric" => Map.get(assertion_params, "assertion_rubric_#{idx}", "")}}
+
+      _other ->
+        {:error, "Assertion at index #{idx}: rubric_judge type requires a valid rubric source"}
+    end
+  end
+
+  defp preview_rubric_source(assertion_params, idx) do
+    case Map.get(assertion_params, "assertion_rubric_source_#{idx}", "template") do
+      "custom" ->
+        %{"rubric" => Map.get(assertion_params, "assertion_rubric_#{idx}", "")}
+
+      _other ->
+        %{"template" => Map.get(assertion_params, "assertion_template_#{idx}", "")}
+    end
+  end
+
+  defp parse_threshold("", _idx, _type) do
+    {:ok, nil}
+  end
+
+  defp parse_threshold(value, idx, type) when is_binary(value) do
     case Float.parse(String.trim(value)) do
       {threshold, ""} ->
         {:ok, threshold}
 
       _other ->
-        {:error,
-         "Assertion at index #{idx}: json_deep_compare type requires a threshold between 0 and 100"}
+        {:error, "Assertion at index #{idx}: #{type} type requires a threshold between 0 and 100"}
     end
   end
 
-  defp parse_threshold(_value, idx) do
-    {:error,
-     "Assertion at index #{idx}: json_deep_compare type requires a threshold between 0 and 100"}
+  defp parse_threshold(_value, idx, type) do
+    {:error, "Assertion at index #{idx}: #{type} type requires a threshold between 0 and 100"}
   end
 
   defp collect_visual_assertion(assertion_params, idx, {:ok, assertions}, mode) do
@@ -391,6 +473,40 @@ defmodule Aludel.Evals.AssertionParser do
   defp maybe_put_threshold(assertion, nil), do: assertion
   defp maybe_put_threshold(assertion, threshold), do: Map.put(assertion, "threshold", threshold)
 
+  defp put_evidence_form_value(params, idx, field, value) do
+    params
+    |> Map.put("assertion_#{field}_#{idx}", format_evidence(value))
+    |> Map.put("assertion_#{field}_json_value_#{idx}", Jason.encode!(value))
+  end
+
+  defp maybe_put_evidence(assertion, assertion_params, idx, field) do
+    text = Map.get(assertion_params, "assertion_#{field}_#{idx}", "")
+    encoded = Map.get(assertion_params, "assertion_#{field}_json_value_#{idx}", "")
+
+    case parse_optional_evidence(text, encoded) do
+      :omit -> assertion
+      {:put, value} -> Map.put(assertion, field, value)
+    end
+  end
+
+  defp parse_optional_evidence(value, _encoded) when value in [nil, ""] do
+    :omit
+  end
+
+  defp parse_optional_evidence(value, encoded) when is_binary(value) do
+    case Jason.decode(encoded) do
+      {:ok, decoded} ->
+        if value == format_evidence(decoded), do: {:put, decoded}, else: {:put, value}
+
+      _other ->
+        {:put, value}
+    end
+  end
+
+  defp parse_optional_evidence(value, _encoded) do
+    {:put, value}
+  end
+
   defp maybe_put_preview_expected(assertion, value) when is_binary(value) do
     case Jason.decode(value) do
       {:ok, decoded} when is_map(decoded) or is_list(decoded) ->
@@ -452,6 +568,30 @@ defmodule Aludel.Evals.AssertionParser do
   defp format_threshold(nil), do: ""
   defp format_threshold(value), do: to_string(value)
 
+  defp rubric_source(%{"rubric" => rubric}) when is_binary(rubric) do
+    "custom"
+  end
+
+  defp rubric_source(_assertion) do
+    "template"
+  end
+
+  defp format_evidence(nil) do
+    ""
+  end
+
+  defp format_evidence(value) when is_binary(value) do
+    value
+  end
+
+  defp format_evidence(value) when is_map(value) or is_list(value) do
+    Jason.encode!(value)
+  end
+
+  defp format_evidence(value) do
+    to_string(value)
+  end
+
   defp blank_string?(value) when is_binary(value), do: String.trim(value) == ""
   defp blank_string?(_value), do: false
 end

lib/aludel/web/live/suite_live/new.ex

@@ -8,9 +8,11 @@ defmodule Aludel.Web.SuiteLive.New do
   alias Aludel.Evals
   alias Aludel.Evals.AssertionParser
   alias Aludel.Evals.DocumentIngestion
+  alias Aludel.Evals.JudgeCatalog
   alias Aludel.Evals.Suite
   alias Aludel.Projects
   alias Aludel.Prompts
+  alias Aludel.Providers
 
   @document_upload_names Enum.map(0..49, &:"test_case_documents_#{&1}")
   @document_upload_accept ~w(.pdf .png .jpg .jpeg .csv .json .txt)
@@ -157,13 +159,16 @@ defmodule Aludel.Web.SuiteLive.New do
     changeset = Evals.change_suite(suite, initial_data)
     prompts = Prompts.list_prompts_with_versions()
     projects = Projects.list_projects(type: :suite)
+    providers = Providers.list_providers()
 
     socket
     |> assign(:page_title, "New Suite")
     |> assign(:suite, suite)
     |> assign(:form, to_form(changeset))
     |> assign(:prompts, prompts)
     |> assign(:projects, projects)
+    |> assign(:providers, providers)
+    |> assign(:judge_templates, JudgeCatalog.all())
     |> assign(:test_cases, [])
     |> assign(:selected_prompt, nil)
     |> assign(:assertion_edit_mode, %{})
@@ -175,6 +180,7 @@ defmodule Aludel.Web.SuiteLive.New do
     changeset = Evals.change_suite(suite)
     prompts = Prompts.list_prompts_with_versions()
     projects = Projects.list_projects(type: :suite)
+    providers = Providers.list_providers()
 
     # Get the selected prompt if suite has one
     selected_prompt =
@@ -202,6 +208,8 @@ defmodule Aludel.Web.SuiteLive.New do
     |> assign(:form, to_form(changeset))
     |> assign(:prompts, prompts)
     |> assign(:projects, projects)
+    |> assign(:providers, providers)
+    |> assign(:judge_templates, JudgeCatalog.all())
     |> assign(:test_cases, test_cases)
     |> assign(:selected_prompt, selected_prompt)
     |> assign(:assertion_edit_mode, %{})
@@ -614,6 +622,36 @@ defmodule Aludel.Web.SuiteLive.New do
     end
   end
 
+  defp assertion_rubric_source_value(test_case_id, idx, form_params, assertion) do
+    case assertion_form_value(test_case_id, idx, "rubric_source", form_params) do
+      nil -> if is_binary(assertion["rubric"]), do: "custom", else: "template"
+      value -> value
+    end
+  end
+
+  defp assertion_evidence_json_value(
+         test_case_id,
+         idx,
+         field_name,
+         form_params,
+         assertion
+       ) do
+    case assertion_form_value(test_case_id, idx, "#{field_name}_json_value", form_params) do
+      nil -> Jason.encode!(Map.get(assertion, field_name))
+      value -> value
+    end
+  end
+
+  defp judge_provider_options(providers, selected_id) do
+    options = Enum.map(providers, &{&1.name, &1.id})
+
+    if selected_id == "" or Enum.any?(providers, &(&1.id == selected_id)) do
+      options
+    else
+      [{"Unavailable provider (#{selected_id})", selected_id} | options]
+    end
+  end
+
   defp assertion_form_value(test_case_id, idx, field_name, form_params) do
     form_params
     |> Map.get(test_case_id, %{})