comet-ml · natagh23 · Jul 22, 2026 · Jul 22, 2026 · Jul 22, 2026 · Jul 22, 2026
@@ -17,6 +17,8 @@ export interface PythonSdkClient {
     tags?: string[];
     thread_id?: string;
     feedback_scores?: Array<{ name: string; value: number; reason?: string }>;
+    error_info?: { exception_type: string; message: string; traceback?: string };
+    duration_seconds?: number;
     spans: Array<{
       name: string;
       type?: 'general' | 'llm' | 'tool';

@@ -0,0 +1,139 @@
+import { test as baseTest } from './annotation-queue.fixture';
+
+export interface ExplainTraceRef {
+  id: string;
+  name: string;
+  errorType: string;
+  /** Seeded trace duration in seconds; null means left unset (renders as Duration "NA"). */
+  durationSeconds: number | null;
+  /** Seeded LLM-span cost; null means no cost-bearing span (renders as Cost "-"). */
+  cost: number | null;
+  /**
+   * Source of a case-insensitive RegExp expected to appear in Ollie's error
+   * explanation for this trace. Ollie's phrasing is non-deterministic (see
+   * ollie-explain.spec.ts), so this is anchored to concrete details of the
+   * seeded error (exception type / message) rather than generic
+   * error-adjacent vocabulary ("error", "fail", ...) — those turned out to be
+   * skippable depending on phrasing (e.g. a rate-limit explanation that never
+   * says "fail" or "exceed"), while the concrete subject (rate limit,
+   * document store, timeout, permissions, faiss) reliably recurs because
+   * Ollie is grounded in that seeded content.
+   */
+  errorKeywordSource: string;
+}
+
+export interface ExplainTracesFixtures {
+  explainTraces: ExplainTraceRef[];
+}
+
+/**
+ * Five traces shaped for the Ollie "Explain" cell feature: each has a distinct
+ * trace-level error, and distinct duration/cost, with one trace leaving both
+ * duration and cost unset so the explain button's N/A path (Duration "NA",
+ * Cost "-") gets covered too — per `explainTargets.ts`, those cells stay
+ * explainable even at N/A, unlike the error cell which vetoes on no error.
+ */
+const SEEDS: Array<{
+  suffix: string;
+  errorType: string;
+  errorMessage: string;
+  /** RegExp source (case-insensitive) expected in Ollie's explanation of this error. */
+  errorKeyword: string;
+  durationSeconds: number | null;
+  cost: number | null;
+}> = [
+  {
+    suffix: 'rate-limit',
+    errorType: 'RuntimeError',
+    errorMessage: 'Model returned status 429: rate limit exceeded',
+    errorKeyword: 'rate.?limit',
+    durationSeconds: 2,
+    cost: 0.0005,
+  },
+  {
+    suffix: 'missing-context',
+    errorType: 'ValueError',
+    errorMessage: 'Missing required context: document store unavailable',
+    errorKeyword: 'context|document store',
+    durationSeconds: 15,
+    cost: 0.02,
+  },
+  {
+    suffix: 'tool-timeout',
+    errorType: 'TimeoutError',
+    errorMessage: 'Tool call timed out after 30 s',
+    errorKeyword: 'timeout',
+    durationSeconds: 60,
+    cost: 0.5,
+  },
+  {
+    suffix: 'auth-failure',
+    errorType: 'PermissionError',
+    errorMessage: 'API key does not have access to model claude-3-opus',
+    errorKeyword: 'permission|access|api key',
+    durationSeconds: 180,
+    cost: 2.0,
+  },
+  {
+    suffix: 'quota-exceeded',
+    errorType: 'ImportError',
+    errorMessage: "Required dependency 'faiss' is not installed",
+    errorKeyword: 'faiss|install|depend',
+    durationSeconds: null,
+    cost: null,
+  },
+];
+
+export const test = baseTest.extend<ExplainTracesFixtures>({
+  explainTraces: async ({ sdkClient, project, testNamespace }, use, testInfo) => {
+    const refs: ExplainTraceRef[] = [];
+    for (const seed of SEEDS) {
+      const name = `${testNamespace}-explain-${seed.suffix}`;
+      const spans =
+        seed.cost === null
+          ? []
+          : [
+              {
+                name: 'llm-call',
+                type: 'llm' as const,
+                model: 'gpt-4o',
+                provider: 'openai',
+                input: { prompt: 'seed prompt' },
+                output: { completion: 'seed completion' },
+                usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+                total_cost: seed.cost,
+              },
+            ];
+      const created = await sdkClient.python.createNestedTrace({
+        project_name: project.name,
+        name,
+        input: { user: `trigger for ${seed.suffix}` },
+        tags: ['explain', seed.suffix],
+        error_info: {
+          exception_type: seed.errorType,
+          message: seed.errorMessage,
+        },
+        duration_seconds: seed.durationSeconds ?? undefined,
+        spans,
+      });
+      refs.push({
+        id: created.id,
+        name: created.name,
+        errorType: seed.errorType,
+        durationSeconds: seed.durationSeconds,
+        cost: seed.cost,
+        errorKeywordSource: seed.errorKeyword,
+      });
+    }
+
+    await testInfo.attach('opik.explainTraces', {
+      body: JSON.stringify(refs, null, 2),
+      contentType: 'application/json',
+    });
+
+    await use(refs);
+    // No explicit teardown — the project fixture's deleteProject cascades.
+  },
+});
+
+export { expect } from './annotation-queue.fixture';
@@ -1,4 +1,4 @@
-export { test, expect } from './annotation-queue.fixture';
+export { test, expect } from './explain-traces.fixture';
 export type { ProjectFixtures } from './project.fixture';
 export type { ScratchDir, ScratchDirFixtures } from './scratch-dir.fixture';
 export type {
@@ -45,4 +45,5 @@ export type {
   AnnotationQueueTraceRef,
   AnnotationQueueFixtures,
 } from './annotation-queue.fixture';
+export type { ExplainTraceRef, ExplainTracesFixtures } from './explain-traces.fixture';
 export type { ProjectRef } from '../core/backend';
@@ -1,8 +1,23 @@
-import { test, type Page, type Locator } from '@playwright/test';
+import { test, expect, type Page, type Locator } from '@playwright/test';
 import { loadEnvConfig } from '../config/env.config';
 import { TracePanelPage } from './trace-panel.page';
 import { ThreadPanelPage } from './thread-panel.page';
 
+export type ExplainKind = 'error' | 'duration' | 'cost';
+
+// Maps an explain kind to the Traces table column id (used in data-cell-id)
+// and the owl trigger's aria-label, per apps/opik-frontend/src/plugins/comet/explain/registry.ts.
+const EXPLAIN_COLUMN: Record<ExplainKind, string> = {
+  error: 'error_info',
+  duration: 'duration',
+  cost: 'total_estimated_cost',
+};
+const EXPLAIN_LABEL: Record<ExplainKind, string> = {
+  error: 'Explain error',
+  duration: 'Explain duration',
+  cost: 'Explain cost',
+};
+
 export class LogsPage {
   private projectId: string | null = null;
 
@@ -116,6 +131,80 @@ export class LogsPage {
     return this.page.locator('tr[data-row-id]');
   }
 
+  /** The Errors/Duration/Estimated cost cell for a trace row, keyed by Ollie explain kind. */
+  explainCell(traceId: string, kind: ExplainKind): Locator {
+    return this.page.locator(`[data-cell-id="${traceId}_${EXPLAIN_COLUMN[kind]}"]`);
+  }
+
+  /**
+   * Hover a trace's Errors/Duration/Estimated cost cell and click its Ollie
+   * "Explain" owl trigger, opening the popover. The trigger only renders once
+   * the Ollie assistant bridge handshake (mounted via the page's assistant
+   * sidebar) completes, which can lag a beat after the table itself is
+   * interactive — so this polls hover+lookup rather than asserting once.
+   */
+  async openExplain(traceId: string, kind: ExplainKind, timeoutMs = 60_000): Promise<void> {
+    return test.step(`open Ollie explain (${kind}) for trace ${traceId}`, async () => {
+      const cell = this.explainCell(traceId, kind);
+      const button = cell.getByRole('button', { name: EXPLAIN_LABEL[kind] });
+      await expect
+        .poll(
+          async () => {
+            await cell.hover();
+            return button.count();
+          },
+          { timeout: timeoutMs, intervals: [500, 1000, 2000] },
+        )
+        .toBeGreaterThan(0);
+      await button.click();
+    });
+  }
+
+  /**
+   * Wait for the open Ollie explain popover to settle (loading -> done/error)
+   * and return its rendered text. Scoped to the last `[role="status"]` live
+   * region on the page — Radix unmounts a closed popover's content, so only
+   * the currently-open one's region should be present.
+   */
+  async readExplanation(timeoutMs = 60_000): Promise<string> {
+    return test.step('wait for Ollie explain popover to settle', async () => {
+      const status = this.page.locator('[role="status"]').last();
+      await expect(status).toHaveAttribute('aria-busy', 'false', { timeout: timeoutMs });
+      const text = ((await status.textContent()) ?? '').trim();
+      if (!text) {
+        throw new Error('Ollie explain popover settled but rendered no text');
+      }
+      return text;
+    });
+  }
+
+  /** Close the open Ollie explain popover. */
+  async closeExplain(): Promise<void> {
+    return test.step('close Ollie explain popover', async () => {
+      await this.page.keyboard.press('Escape');
+    });
+  }
+
+  /**
+   * The "Continue conversation" link in the currently open Ollie explain
+   * popover. Only rendered once the popover has settled with text (see
+   * ExplainPopover.tsx) — call after `readExplanation()`.
+   */
+  continueConversationButton(): Locator {
+    return this.page.getByRole('button', { name: 'Continue conversation' });
+  }
+
+  /**
+   * Click "Continue conversation" to hand the explain popover's question +
+   * cached answer off to the Ollie sidebar chat. This closes the popover as
+   * a side effect (see ExplainPopover's onContinue).
+   */
+  async continueConversation(): Promise<void> {
+    return test.step('continue the Ollie explain conversation in the sidebar', async () => {
+      await this.continueConversationButton().click();
+    });
+  }
+
   // --- Threads tab ---
 
   /** The Threads/Traces/Spans tab toggle for "Threads". */

@@ -128,6 +128,36 @@ export class OlliePage {
     });
   }
 
+  /**
+   * Wait for an Explain popover's "Continue conversation" hand-off to land in
+   * the chat. The bridge posts the question and the popover's already-settled
+   * answer as a pair of new messages (see `chat:continue` in explainStore.ts
+   * — "carries the verbatim Q&A already shown"), not a fresh generation, so
+   * this only waits for them to render, not for streaming. `beforeCount` is
+   * the message count read just before clicking "Continue conversation".
+   * Returns the last message's (the answer's) text.
+   */
+  async awaitContinuedConversation(beforeCount: number, timeoutMs = 30_000): Promise<string> {
+    return test.step('wait for the continued conversation to render in the sidebar', async () => {
+      await expect
+        .poll(async () => this.messages().count(), {
+          timeout: timeoutMs,
+          intervals: [300, 600, 1200],
+        })
+        .toBeGreaterThanOrEqual(beforeCount + 2);
+
+      const reply = this.messages().last();
+      await expect
+        .poll(async () => ((await reply.textContent()) ?? '').trim().length, {
+          timeout: timeoutMs,
+          intervals: [300, 600, 1200],
+        })
+        .toBeGreaterThan(0);
+
+      return ((await reply.textContent()) ?? '').trim();
+    });
+  }
+
   /**
    * Run the `/analyze` flow from the greeting action button and wait for a
    * non-empty assistant response to render. Ollie is a non-deterministic agent,

@@ -1,4 +1,5 @@
 import atexit
+import datetime
 
 import opik
 from fastapi import APIRouter, Header, HTTPException
@@ -76,6 +77,12 @@ def create_nested_trace(
     # SDK is translated to HTTP by the app-wide exception handler.
     client = make_opik_client(workspace=body.workspace, api_key=x_opik_api_key)
     try:
+        start_time: datetime.datetime | None = None
+        end_time: datetime.datetime | None = None
+        if body.duration_seconds is not None:
+            end_time = datetime.datetime.now(datetime.timezone.utc)
+            start_time = end_time - datetime.timedelta(seconds=body.duration_seconds)
+
         trace = client.trace(
             project_name=body.project_name,
             name=body.name,
@@ -85,6 +92,18 @@ def create_nested_trace(
             tags=body.tags,
             thread_id=body.thread_id,
             feedback_scores=body.feedback_scores,
+            error_info=(
+                {
+                    "exception_type": body.error_info.exception_type,
+                    "message": body.error_info.message,
+                    # The REST ErrorInfo type requires a non-null traceback string.
+                    "traceback": body.error_info.traceback or "",
+                }
+                if body.error_info
+                else None
+            ),
+            start_time=start_time,
+            end_time=end_time,
         )
 
         created: list = []

@@ -56,6 +56,14 @@ class SpanSeed(BaseModel):
     parent_index: int | None = None
 
 
+class ErrorInfoSeed(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    exception_type: str
+    message: str
+    traceback: str | None = None
+
+
 class NestedTraceCreate(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -69,6 +77,14 @@ class NestedTraceCreate(BaseModel):
     feedback_scores: list[dict[str, Any]] | None = None
     spans: list[SpanSeed]
     workspace: str | None = None
+    # Sets the trace's own error_info (as opposed to a span's), driving the
+    # Traces table's Errors column/explain target. None means no trace-level error.
+    error_info: ErrorInfoSeed | None = None
+    # Backdates start_time by this many seconds and sets end_time to now, so the
+    # trace renders a specific Duration cell value. None leaves both start_time
+    # and end_time unset, which the UI renders as Duration "NA" — the same shape
+    # as the SDK's own not-yet-ended traces.
+    duration_seconds: float | None = None
 
 
 class NestedTraceResponse(BaseModel):