[OPIK-8102] [QA] Proposed e2e specs for the LLM-judge Gemini thinking level (from #8158 exploration)

apps/opik-frontend/src/v2/pages-shared/llm/PromptModelSettings/PromptModelConfigs.tsx

@@ -122,7 +122,15 @@ const PromptModelConfigs = ({
     <DropdownMenu>
       <TooltipWrapper content="Model parameters">
         <DropdownMenuTrigger asChild>
-          <Button variant={variant} size={size} disabled={disabled}>
+          {/* Icon-only, so it has no accessible name to address it by — the
+              "Model parameters" string lives in the tooltip, which does not
+              contribute one. Test-only hook; nothing reads it at runtime. */}
+          <Button
+            variant={variant}
+            size={size}
+            disabled={disabled}
+            data-testid="model-parameters-trigger"
+          >
             <Settings2 />
           </Button>
         </DropdownMenuTrigger>

tests_end_to_end/coverage/taxonomy.yaml

@@ -648,8 +648,9 @@ areas:
       - online-evaluation/online-evaluation-sampling-rate.spec.ts
       - online-evaluation/online-evaluation-python-metric-errors.spec.ts
       - online-evaluation/online-evaluation-non-object-sections.spec.ts
+      - online-evaluation/online-evaluation-thinking-level.spec.ts
     capabilities:
-      create-llm-judge-rule:   { covered: true,  tier: t1-smoke }
+      create-llm-judge-rule:   { covered: true,  tier: t1-smoke, note: "t2 also covers creation on a Gemini model, asserting the model-settings pane's preselected thinking level and offered levels against what the save persists" }
       llm-judge-scores:        { covered: true,  tier: t1-smoke, note: "bimodal safe/unsafe" }
       create-python-rule:      { covered: true,  tier: t1-smoke }
       python-rule-scores:      { covered: true,  tier: t1-smoke, note: "deterministic 3x1.0 / 2x0.0; t2 also covers sub-path mappings over non-object sections and the 400-class classification of a metric that exits 0 without a result line" }
@@ -659,7 +660,7 @@ areas:
       rule-filters:            { covered: false }
       sampling-rate:           { covered: true,  tier: t2-cuj, note: "50% rule vs 100% control over one 30-trace batch, binomial band 15-85%; plus a 0%-rate rule at trigger_scope=both, which must skip every SDK trace and still score experiment/playground/optimization ones" }
       clone-rule:              { covered: false }
-      edit-rule:               { covered: false }
+      edit-rule:               { covered: true,  tier: t2-cuj, note: "row kebab -> Edit -> submit, asserted on the persisted evaluator: the Gemini thinking level round-trips (saved -> rehydrated -> resaved), an untouched resave preserves it, and setting it back to None removes the block rather than leaving it behind. Only the model-settings pane of the dialog; the prompt, variable mapping, filters and sampling controls are still unexercised by an edit" }
       enable-disable-rule:     { covered: true,  tier: t2-cuj, note: "edit-dialog switch; control rule proves scoring stopped, then resumed" }
       delete-rule:             { covered: true,  tier: t2-cuj, note: "row kebab delete; control rule proves scoring stopped" }
       # Deliberately still false. online-evaluation-python-metric-errors.spec.ts

tests_end_to_end/e2e/core/backend/client.ts

@@ -181,6 +181,28 @@ export interface AutomationRuleDetail {
   triggerScope: string;
 }
 
+/**
+ * The `code.model` block of an LLM-as-judge rule, as the REST view answers it.
+ *
+ * Read raw rather than through the pinned SDK for the same reason as
+ * `AutomationRuleDetail`: the SDK's evaluator shape has no
+ * `custom_parameters`, and this is the field that decides what the judge call
+ * actually sends to the provider.
+ */
+export interface LlmJudgeModelDetail {
+  /** The model id the rule will call, e.g. `gemini-3.5-flash-lite`. */
+  name: string;
+  /**
+   * `custom_parameters` verbatim, `null` when the rule carries none.
+   *
+   * Kept nullable rather than defaulted to `{}` because absent and empty are
+   * different answers here: a thinking level of "none" must REMOVE the block,
+   * and a spec that cannot tell "no block" from "empty block" cannot assert
+   * that.
+   */
+  customParameters: Record<string, unknown> | null;
+}
+
 /** One line of a rule's user-facing log stream. */
 export interface AutomationRuleLogRef {
   level: string;
@@ -1580,6 +1602,41 @@ export function makeBackendClient(apiKey: string | null = null, workspaceName: s
       };
     },
 
+    /**
+     * The model block of an LLM-as-judge rule — the model id and the
+     * `custom_parameters` the judge call will carry.
+     *
+     * Throws rather than defaulting when the rule is not an LLM-as-judge one:
+     * a Python-code rule has no `code.model` at all, and answering `null` for
+     * it would let a mis-typed spec assert "no custom parameters" about a rule
+     * that could never have had any.
+     */
+    async getLlmJudgeModel(ruleId: string): Promise<LlmJudgeModelDetail> {
+      const { status, message, json } = await rawFetch(
+        'GET',
+        `/v1/private/automations/evaluators/${ruleId}`,
+      );
+      if (status !== 200) {
+        throw new Error(`getLlmJudgeModel: ${ruleId} answered ${status}: ${message}`);
+      }
+      const model = (json as { code?: { model?: unknown } } | null)?.code?.model as
+        | { name?: unknown; custom_parameters?: unknown }
+        | undefined;
+      if (!model || typeof model.name !== 'string') {
+        throw new Error(
+          `getLlmJudgeModel: ${ruleId} carries no code.model.name — not an LLM-as-judge rule?`,
+        );
+      }
+      const custom = model.custom_parameters;
+      return {
+        name: model.name,
+        customParameters:
+          custom === null || custom === undefined
+            ? null
+            : (custom as Record<string, unknown>),
+      };
+    },
+
     /**
      * A rule's user-facing log stream — the lines `/automation-logs` renders.
      *

tests_end_to_end/e2e/core/backend/index.ts

@@ -26,6 +26,7 @@ export {
   type AutomationRuleRef,
   type AutomationRuleDetail,
   type AutomationRuleLogRef,
+  type LlmJudgeModelDetail,
   type TraceJsonSection,
   type AnnotationQueueDetail,
   type AnnotationQueueReviewerRef,

tests_end_to_end/e2e/core/provider-keys.ts

@@ -49,20 +49,52 @@ function endpoint(path = ''): string {
   return `${env.apiBaseUrl}/v1/private/llm-provider-key${path}`;
 }
 
-export async function findProviderKeyByName(providerName: string): Promise<ProviderKeyRef | null> {
+async function listProviderKeys(): Promise<ProviderKeyRef[]> {
   const response = await fetch(endpoint(), { headers: restHeaders() });
   if (!response.ok) throw new Error(`list provider keys returned ${response.status}`);
   const body = (await response.json()) as { content: ProviderKeyRef[] };
-  return body.content.find((key) => key.provider_name === providerName) ?? null;
+  return body.content;
+}
+
+export async function findProviderKeyByName(providerName: string): Promise<ProviderKeyRef | null> {
+  const keys = await listProviderKeys();
+  return keys.find((key) => key.provider_name === providerName) ?? null;
 }
 
+/**
+ * Find a key by its PROVIDER slug (`gemini`, `vertex-ai`, …) rather than by
+ * `provider_name`, which only the custom/bedrock/ollama providers carry.
+ *
+ * The workspace holds at most one key per built-in provider, so this is the
+ * only way to ask "is Gemini already configured here" — and a caller that
+ * seeds one has to ask, because creating a second would collide with whatever
+ * key the environment already had.
+ */
+export async function findProviderKeyByProvider(provider: string): Promise<ProviderKeyRef | null> {
+  const keys = await listProviderKeys();
+  return keys.find((key) => key.provider === provider) ?? null;
+}
+
+/**
+ * Create a provider key, and report its id from the `Location` header —
+ * creation answers 201 with an empty body, so that header is the only place
+ * the id appears.
+ *
+ * `null` rather than a throw when the header is missing, so a caller that
+ * cleans up by `provider_name` is not made to care: only a caller that has to
+ * address the key by id needs the header, and that caller is the one that
+ * should complain about it.
+ */
 export async function createProviderKey(payload: {
   provider: string;
-  provider_name: string;
-  base_url: string;
+  /** Only the custom/bedrock/ollama providers name their keys. */
+  provider_name?: string;
+  base_url?: string;
+  /** The provider secret. Built-in providers key off this rather than `auth_config`. */
+  api_key?: string;
   configuration?: Record<string, string>;
   auth_config?: ProviderAuthConfig;
-}): Promise<void> {
+}): Promise<string | null> {
   const response = await fetch(endpoint(), {
     method: 'POST',
     headers: restHeaders(),
@@ -71,21 +103,26 @@ export async function createProviderKey(payload: {
   if (!response.ok) {
     throw new Error(`create provider key returned ${response.status}: ${await response.text()}`);
   }
+  return response.headers.get('location')?.split('/').filter(Boolean).pop() ?? null;
 }
 
-export async function deleteProviderKeyByName(providerName: string): Promise<void> {
-  const found = await findProviderKeyByName(providerName);
-  if (!found) return;
+export async function deleteProviderKeyById(id: string): Promise<void> {
   const response = await fetch(endpoint('/delete'), {
     method: 'POST',
     headers: restHeaders(),
-    body: JSON.stringify({ ids: [found.id] }),
+    body: JSON.stringify({ ids: [id] }),
   });
   if (!response.ok) {
     throw new Error(`delete provider key returned ${response.status}`);
   }
 }
 
+export async function deleteProviderKeyByName(providerName: string): Promise<void> {
+  const found = await findProviderKeyByName(providerName);
+  if (!found) return;
+  await deleteProviderKeyById(found.id);
+}
+
 /** Carries the HTTP status so callers can classify a failure instead of parsing prose. */
 export class AuthConfigCheckError extends Error {
   constructor(

tests_end_to_end/e2e/fixtures/index.ts

@@ -3,6 +3,7 @@ export type {
   OauthProviderSeed,
   ProviderKeysFixture,
   ProviderKeyFixtures,
+  GeminiProviderKeyRef,
 } from './provider-key.fixture';
 export type { ProjectFixtures } from './project.fixture';
 export type {

tests_end_to_end/e2e/fixtures/provider-key.fixture.ts

@@ -6,7 +6,12 @@ import {
   mockGatewayUrlForBackend,
   mockTokenUrlForBackend,
 } from '../core/mock-auth';
-import { createProviderKey, deleteProviderKeyByName } from '../core/provider-keys';
+import {
+  createProviderKey,
+  deleteProviderKeyById,
+  deleteProviderKeyByName,
+  findProviderKeyByProvider,
+} from '../core/provider-keys';
 
 export interface OauthProviderSeed {
   providerName: string;
@@ -27,10 +32,37 @@ export interface ProviderKeysFixture {
   register(providerName: string): void;
 }
 
+export interface GeminiProviderKeyRef {
+  /** The provider slug the API and the model picker agree on. */
+  provider: 'gemini';
+  /** The picker's group label for this provider — the scope a model lookup needs. */
+  groupLabel: 'Gemini';
+  /**
+   * True when this fixture created the key (and will delete it), false when it
+   * adopted one the environment already had (and will leave it alone).
+   */
+  seeded: boolean;
+}
+
 export interface ProviderKeyFixtures {
   providerKeys: ProviderKeysFixture;
+  /**
+   * Makes the Gemini models selectable in every model picker, without calling
+   * an LLM: the key holds a dummy secret, which is enough for the FE — the
+   * picker lists a provider's models as soon as a key for it exists.
+   *
+   * Only for specs that assert on FORM state and on what a save persists. A
+   * spec that needs a completion needs a real key and must not use this.
+   */
+  geminiProviderKey: GeminiProviderKeyRef;
 }
 
+/**
+ * Not a real credential, and deliberately shaped so that it reads as one in a
+ * request log: nothing this fixture supports ever reaches Google.
+ */
+const DUMMY_GEMINI_API_KEY = 'e2e-dummy-key-no-completions-are-made-with-this';
+
 /**
  * Provider keys are WORKSPACE-GLOBAL, so every spec must use testNamespace-prefixed
  * names and delete what it creates — this fixture owns the delete half.
@@ -74,6 +106,50 @@ export const test = baseTest.extend<ProviderKeyFixtures>({
       }
     }
   },
+
+  // eslint-disable-next-line no-empty-pattern
+  geminiProviderKey: async ({}, use, testInfo) => {
+    // A workspace holds at most one key per built-in provider and the name
+    // cannot be namespaced the way `providerKeys.createOauth` namespaces a
+    // custom provider, so this is adopt-or-seed rather than seed: if the
+    // environment already has a Gemini key (a real one on a CI workspace, say)
+    // it is used as-is and left in place. Deleting a key this fixture did not
+    // create would break whatever configured it.
+    //
+    // Because the resource is workspace-global and unnamespaced, specs using
+    // this fixture must not run concurrently with each other — see the serial
+    // mode on online-evaluation-thinking-level.spec.ts.
+    //
+    // The other writer of this same key is playground-providers.spec.ts, which
+    // self-provisions Gemini through the AI Providers UI and then wants a real
+    // completion from it. The two do not collide today because that spec is
+    // @provider-sanity, which runs on its own cadence rather than in the tier
+    // ladder — but running both at once would let it adopt this dummy key and
+    // fail on auth. Keep them in separate runs.
+    let createdId: string | null = null;
+    if (!(await findProviderKeyByProvider('gemini'))) {
+      createdId = await createProviderKey({ provider: 'gemini', api_key: DUMMY_GEMINI_API_KEY });
+      if (createdId === null) {
+        // Refusing to continue rather than leaking: without the id this
+        // fixture cannot delete the key it just created, and a stray Gemini
+        // key changes what every later run's model picker offers.
+        throw new Error(
+          'gemini provider key was created but answered no Location header — ' +
+            'it cannot be addressed for teardown',
+        );
+      }
+    }
+
+    await use({ provider: 'gemini', groupLabel: 'Gemini', seeded: createdId !== null });
+
+    if (createdId !== null && !shouldLeaveArtifacts(testInfo)) {
+      try {
+        await deleteProviderKeyById(createdId);
+      } catch (err) {
+        console.warn('[provider-key fixture] gemini key delete warning:', err);
+      }
+    }
+  },
 });
 
 export { expect } from './automation-rules.fixture';