From 64e7ce52ffebeb0d042c7430c80239cb9a68bde9 Mon Sep 17 00:00:00 2001 From: natagh23 Date: Wed, 22 Jul 2026 12:32:19 +0200 Subject: [PATCH 1/4] test for Explain Ollie feature --- .../e2e/core/sdk/python-sdk-client.ts | 2 + .../e2e/fixtures/explain-traces.fixture.ts | 139 ++++++++++++++++++ tests_end_to_end/e2e/fixtures/index.ts | 3 +- tests_end_to_end/e2e/pom/logs.page.ts | 91 +++++++++++- tests_end_to_end/e2e/pom/ollie.page.ts | 30 ++++ .../src/opik_sdk_driver/routes/traces.py | 19 +++ .../src/opik_sdk_driver/schemas.py | 16 ++ .../e2e/tests/ollie/ollie-explain.spec.ts | 111 ++++++++++++++ 8 files changed, 409 insertions(+), 2 deletions(-) create mode 100644 tests_end_to_end/e2e/fixtures/explain-traces.fixture.ts create mode 100644 tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts diff --git a/tests_end_to_end/e2e/core/sdk/python-sdk-client.ts b/tests_end_to_end/e2e/core/sdk/python-sdk-client.ts index ece225cc4d4..746ccc54d51 100644 --- a/tests_end_to_end/e2e/core/sdk/python-sdk-client.ts +++ b/tests_end_to_end/e2e/core/sdk/python-sdk-client.ts @@ -17,6 +17,8 @@ export interface PythonSdkClient { tags?: string[]; thread_id?: string; feedback_scores?: Array<{ name: string; value: number; reason?: string }>; + error_info?: { exception_type: string; message: string; traceback?: string }; + duration_seconds?: number; spans: Array<{ name: string; type?: 'general' | 'llm' | 'tool'; diff --git a/tests_end_to_end/e2e/fixtures/explain-traces.fixture.ts b/tests_end_to_end/e2e/fixtures/explain-traces.fixture.ts new file mode 100644 index 00000000000..f60772c6d58 --- /dev/null +++ b/tests_end_to_end/e2e/fixtures/explain-traces.fixture.ts @@ -0,0 +1,139 @@ +import { test as baseTest } from './annotation-queue.fixture'; + +export interface ExplainTraceRef { + id: string; + name: string; + errorType: string; + /** Seeded trace duration in seconds; null means left unset (renders as Duration "NA"). */ + durationSeconds: number | null; + /** Seeded LLM-span cost; null means no cost-bearing span (renders as Cost "-"). */ + cost: number | null; + /** + * Source of a case-insensitive RegExp expected to appear in Ollie's error + * explanation for this trace. Ollie's phrasing is non-deterministic (see + * ollie-explain.spec.ts), so this is anchored to concrete details of the + * seeded error (exception type / message) rather than generic + * error-adjacent vocabulary ("error", "fail", ...) — those turned out to be + * skippable depending on phrasing (e.g. a rate-limit explanation that never + * says "fail" or "exceed"), while the concrete subject (rate limit, + * document store, timeout, permissions, faiss) reliably recurs because + * Ollie is grounded in that seeded content. + */ + errorKeywordSource: string; +} + +export interface ExplainTracesFixtures { + explainTraces: ExplainTraceRef[]; +} + +/** + * Five traces shaped for the Ollie "Explain" cell feature: each has a distinct + * trace-level error, and distinct duration/cost, with one trace leaving both + * duration and cost unset so the explain button's N/A path (Duration "NA", + * Cost "-") gets covered too — per `explainTargets.ts`, those cells stay + * explainable even at N/A, unlike the error cell which vetoes on no error. + */ +const SEEDS: Array<{ + suffix: string; + errorType: string; + errorMessage: string; + /** RegExp source (case-insensitive) expected in Ollie's explanation of this error. */ + errorKeyword: string; + durationSeconds: number | null; + cost: number | null; +}> = [ + { + suffix: 'rate-limit', + errorType: 'RuntimeError', + errorMessage: 'Model returned status 429: rate limit exceeded', + errorKeyword: 'rate.?limit', + durationSeconds: 2, + cost: 0.0005, + }, + { + suffix: 'missing-context', + errorType: 'ValueError', + errorMessage: 'Missing required context: document store unavailable', + errorKeyword: 'context|document store', + durationSeconds: 15, + cost: 0.02, + }, + { + suffix: 'tool-timeout', + errorType: 'TimeoutError', + errorMessage: 'Tool call timed out after 30 s', + errorKeyword: 'timeout', + durationSeconds: 60, + cost: 0.5, + }, + { + suffix: 'auth-failure', + errorType: 'PermissionError', + errorMessage: 'API key does not have access to model claude-3-opus', + errorKeyword: 'permission|access|api key', + durationSeconds: 180, + cost: 2.0, + }, + { + suffix: 'quota-exceeded', + errorType: 'ImportError', + errorMessage: "Required dependency 'faiss' is not installed", + errorKeyword: 'faiss|install|depend', + durationSeconds: null, + cost: null, + }, +]; + +export const test = baseTest.extend({ + explainTraces: async ({ sdkClient, project, testNamespace }, use, testInfo) => { + const refs: ExplainTraceRef[] = []; + for (const seed of SEEDS) { + const name = `${testNamespace}-explain-${seed.suffix}`; + const spans = + seed.cost === null + ? [] + : [ + { + name: 'llm-call', + type: 'llm' as const, + model: 'gpt-4o', + provider: 'openai', + input: { prompt: 'seed prompt' }, + output: { completion: 'seed completion' }, + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + total_cost: seed.cost, + }, + ]; + const created = await sdkClient.python.createNestedTrace({ + project_name: project.name, + name, + input: { user: `trigger for ${seed.suffix}` }, + tags: ['explain', seed.suffix], + error_info: { + exception_type: seed.errorType, + message: seed.errorMessage, + }, + duration_seconds: seed.durationSeconds ?? undefined, + spans, + }); + refs.push({ + id: created.id, + name: created.name, + errorType: seed.errorType, + durationSeconds: seed.durationSeconds, + cost: seed.cost, + errorKeywordSource: seed.errorKeyword, + }); + } + + await testInfo.attach('opik.explainTraces', { + body: JSON.stringify(refs, null, 2), + contentType: 'application/json', + }); + + await use(refs); + // No explicit teardown — the project fixture's deleteProject cascades. + }, +}); + +export { expect } from './annotation-queue.fixture'; diff --git a/tests_end_to_end/e2e/fixtures/index.ts b/tests_end_to_end/e2e/fixtures/index.ts index 837f6f5688c..8c43d551735 100644 --- a/tests_end_to_end/e2e/fixtures/index.ts +++ b/tests_end_to_end/e2e/fixtures/index.ts @@ -1,4 +1,4 @@ -export { test, expect } from './annotation-queue.fixture'; +export { test, expect } from './explain-traces.fixture'; export type { ProjectFixtures } from './project.fixture'; export type { ScratchDir, ScratchDirFixtures } from './scratch-dir.fixture'; export type { @@ -45,4 +45,5 @@ export type { AnnotationQueueTraceRef, AnnotationQueueFixtures, } from './annotation-queue.fixture'; +export type { ExplainTraceRef, ExplainTracesFixtures } from './explain-traces.fixture'; export type { ProjectRef } from '../core/backend'; diff --git a/tests_end_to_end/e2e/pom/logs.page.ts b/tests_end_to_end/e2e/pom/logs.page.ts index e9976f888fd..a7e2df368ee 100644 --- a/tests_end_to_end/e2e/pom/logs.page.ts +++ b/tests_end_to_end/e2e/pom/logs.page.ts @@ -1,8 +1,23 @@ -import { test, type Page, type Locator } from '@playwright/test'; +import { test, expect, type Page, type Locator } from '@playwright/test'; import { loadEnvConfig } from '../config/env.config'; import { TracePanelPage } from './trace-panel.page'; import { ThreadPanelPage } from './thread-panel.page'; +export type ExplainKind = 'error' | 'duration' | 'cost'; + +// Maps an explain kind to the Traces table column id (used in data-cell-id) +// and the owl trigger's aria-label, per apps/opik-frontend/src/plugins/comet/explain/registry.ts. +const EXPLAIN_COLUMN: Record = { + error: 'error_info', + duration: 'duration', + cost: 'total_estimated_cost', +}; +const EXPLAIN_LABEL: Record = { + error: 'Explain error', + duration: 'Explain duration', + cost: 'Explain cost', +}; + export class LogsPage { private projectId: string | null = null; @@ -116,6 +131,80 @@ export class LogsPage { return this.page.locator('tr[data-row-id]'); } + /** The Errors/Duration/Estimated cost cell for a trace row, keyed by Ollie explain kind. */ + explainCell(traceId: string, kind: ExplainKind): Locator { + return this.page.locator(`[data-cell-id="${traceId}_${EXPLAIN_COLUMN[kind]}"]`); + } + + /** + * Hover a trace's Errors/Duration/Estimated cost cell and click its Ollie + * "Explain" owl trigger, opening the popover. The trigger only renders once + * the Ollie assistant bridge handshake (mounted via the page's assistant + * sidebar) completes, which can lag a beat after the table itself is + * interactive — so this polls hover+lookup rather than asserting once. + */ + async openExplain(traceId: string, kind: ExplainKind, timeoutMs = 60_000): Promise { + return test.step(`open Ollie explain (${kind}) for trace ${traceId}`, async () => { + const cell = this.explainCell(traceId, kind); + const button = cell.getByRole('button', { name: EXPLAIN_LABEL[kind] }); + await expect + .poll( + async () => { + await cell.hover(); + return button.count(); + }, + { timeout: timeoutMs, intervals: [500, 1000, 2000] }, + ) + .toBeGreaterThan(0); + await button.click(); + }); + } + + /** + * Wait for the open Ollie explain popover to settle (loading -> done/error) + * and return its rendered text. Scoped to the last `[role="status"]` live + * region on the page — Radix unmounts a closed popover's content, so only + * the currently-open one's region should be present. + */ + async readExplanation(timeoutMs = 60_000): Promise { + return test.step('wait for Ollie explain popover to settle', async () => { + const status = this.page.locator('[role="status"]').last(); + await expect(status).toHaveAttribute('aria-busy', 'false', { timeout: timeoutMs }); + const text = ((await status.textContent()) ?? '').trim(); + if (!text) { + throw new Error('Ollie explain popover settled but rendered no text'); + } + return text; + }); + } + + /** Close the open Ollie explain popover. */ + async closeExplain(): Promise { + return test.step('close Ollie explain popover', async () => { + await this.page.keyboard.press('Escape'); + }); + } + + /** + * The "Continue conversation" link in the currently open Ollie explain + * popover. Only rendered once the popover has settled with text (see + * ExplainPopover.tsx) — call after `readExplanation()`. + */ + continueConversationButton(): Locator { + return this.page.getByRole('button', { name: 'Continue conversation' }); + } + + /** + * Click "Continue conversation" to hand the explain popover's question + + * cached answer off to the Ollie sidebar chat. This closes the popover as + * a side effect (see ExplainPopover's onContinue). + */ + async continueConversation(): Promise { + return test.step('continue the Ollie explain conversation in the sidebar', async () => { + await this.continueConversationButton().click(); + }); + } + // --- Threads tab --- /** The Threads/Traces/Spans tab toggle for "Threads". */ diff --git a/tests_end_to_end/e2e/pom/ollie.page.ts b/tests_end_to_end/e2e/pom/ollie.page.ts index 5ee203a2738..eca7237fcac 100644 --- a/tests_end_to_end/e2e/pom/ollie.page.ts +++ b/tests_end_to_end/e2e/pom/ollie.page.ts @@ -128,6 +128,36 @@ export class OlliePage { }); } + /** + * Wait for an Explain popover's "Continue conversation" hand-off to land in + * the chat. The bridge posts the question and the popover's already-settled + * answer as a pair of new messages (see `chat:continue` in explainStore.ts + * — "carries the verbatim Q&A already shown"), not a fresh generation, so + * this only waits for them to render, not for streaming. `beforeCount` is + * the message count read just before clicking "Continue conversation". + * Returns the last message's (the answer's) text. + */ + async awaitContinuedConversation(beforeCount: number, timeoutMs = 30_000): Promise { + return test.step('wait for the continued conversation to render in the sidebar', async () => { + await expect + .poll(async () => this.messages().count(), { + timeout: timeoutMs, + intervals: [300, 600, 1200], + }) + .toBeGreaterThanOrEqual(beforeCount + 2); + + const reply = this.messages().last(); + await expect + .poll(async () => ((await reply.textContent()) ?? '').trim().length, { + timeout: timeoutMs, + intervals: [300, 600, 1200], + }) + .toBeGreaterThan(0); + + return ((await reply.textContent()) ?? '').trim(); + }); + } + /** * Run the `/analyze` flow from the greeting action button and wait for a * non-empty assistant response to render. Ollie is a non-deterministic agent, diff --git a/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/routes/traces.py b/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/routes/traces.py index 4a5b7d52c39..7cb972bb925 100644 --- a/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/routes/traces.py +++ b/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/routes/traces.py @@ -1,4 +1,5 @@ import atexit +import datetime import opik from fastapi import APIRouter, Header, HTTPException @@ -76,6 +77,12 @@ def create_nested_trace( # SDK is translated to HTTP by the app-wide exception handler. client = make_opik_client(workspace=body.workspace, api_key=x_opik_api_key) try: + start_time: datetime.datetime | None = None + end_time: datetime.datetime | None = None + if body.duration_seconds is not None: + end_time = datetime.datetime.now(datetime.timezone.utc) + start_time = end_time - datetime.timedelta(seconds=body.duration_seconds) + trace = client.trace( project_name=body.project_name, name=body.name, @@ -85,6 +92,18 @@ def create_nested_trace( tags=body.tags, thread_id=body.thread_id, feedback_scores=body.feedback_scores, + error_info=( + { + "exception_type": body.error_info.exception_type, + "message": body.error_info.message, + # The REST ErrorInfo type requires a non-null traceback string. + "traceback": body.error_info.traceback or "", + } + if body.error_info + else None + ), + start_time=start_time, + end_time=end_time, ) created: list = [] diff --git a/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/schemas.py b/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/schemas.py index 4c95814e6ee..70536747caf 100644 --- a/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/schemas.py +++ b/tests_end_to_end/e2e/services/opik-sdk-driver/src/opik_sdk_driver/schemas.py @@ -56,6 +56,14 @@ class SpanSeed(BaseModel): parent_index: int | None = None +class ErrorInfoSeed(BaseModel): + model_config = ConfigDict(extra="forbid") + + exception_type: str + message: str + traceback: str | None = None + + class NestedTraceCreate(BaseModel): model_config = ConfigDict(extra="forbid") @@ -69,6 +77,14 @@ class NestedTraceCreate(BaseModel): feedback_scores: list[dict[str, Any]] | None = None spans: list[SpanSeed] workspace: str | None = None + # Sets the trace's own error_info (as opposed to a span's), driving the + # Traces table's Errors column/explain target. None means no trace-level error. + error_info: ErrorInfoSeed | None = None + # Backdates start_time by this many seconds and sets end_time to now, so the + # trace renders a specific Duration cell value. None leaves both start_time + # and end_time unset, which the UI renders as Duration "NA" — the same shape + # as the SDK's own not-yet-ended traces. + duration_seconds: float | None = None class NestedTraceResponse(BaseModel): diff --git a/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts b/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts new file mode 100644 index 00000000000..326e9d192b0 --- /dev/null +++ b/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts @@ -0,0 +1,111 @@ +import { test, expect } from '@e2e/fixtures'; +import { LogsPage } from '@e2e/pom/logs.page'; +import { OlliePage } from '@e2e/pom/ollie.page'; + +/** + * Ollie — the per-cell "Explain" owl button on the Traces table (OPIK-6425). + * Cloud/client-only, like the rest of Ollie — see ollie-smoke.spec.ts. + */ +function skipIfOllieDisabled(envConfig: { features: { ollie: boolean } }): void { + test.skip(!envConfig.features.ollie, 'Ollie is cloud/client-only (OLLIE_ENABLED off)'); +} + +// Ollie's wording varies per call (it's a non-deterministic LLM agent), so +// assertions stay topical rather than exact-match. Cost/duration explanations +// are grounded in the same fixed facts regardless of phrasing (a dollar +// figure, a second count), so one shared pattern per kind holds up across +// runs. Error explanations vary far more (e.g. one rate-limit explanation +// never said "fail" or "exceed"), so those are checked per-trace against +// `errorKeywordSource` instead — anchored to the seeded error's concrete +// subject (rate limit, document store, timeout, ...), which Ollie's answer +// reliably references even when the surrounding phrasing differs. +const KEYWORD_PATTERN: Record<'cost' | 'duration', RegExp> = { + cost: /cost/i, + duration: /duration|second|end time|running/i, +}; + +// The popover and the sidebar chat bubble render the same markdown through +// two independent components (the popover per ExplainPopover.tsx; the +// sidebar is the separately-deployed Ollie iframe), so compare on normalized +// text with `toContain` rather than exact equality: the sidebar message's +// textContent also picks up surrounding UI chrome (a status marker, a "Copy" +// button label) that isn't part of the answer itself. +const normalize = (text: string) => text.replace(/\s+/g, ' ').trim(); + +test.describe('Ollie — explain cells', { tag: ['@t2-cuj', '@ollie'] }, () => { + test.beforeEach(({ envConfig }) => skipIfOllieDisabled(envConfig)); + + test('Explain renders on-topic text for the Errors, Estimated cost, and Duration cell of every seeded trace', async ({ + project, + explainTraces, + page, + }) => { + test.setTimeout(600_000); + const logs = new LogsPage(page); + + await test.step('Open Logs and wait for the seeded traces to render', async () => { + await logs.goto(project.id); + await logs.waitForReady(); + expect(await logs.countTraces()).toBe(explainTraces.length); + }); + + for (const trace of explainTraces) { + await test.step(`Explain the Errors cell for "${trace.name}"`, async () => { + await logs.openExplain(trace.id, 'error'); + const text = await logs.readExplanation(); + expect(text.length).toBeGreaterThan(0); + expect(text).toMatch(new RegExp(trace.errorKeywordSource, 'i')); + await logs.closeExplain(); + }); + } + + for (const trace of explainTraces) { + await test.step(`Explain the Estimated cost cell for "${trace.name}" (cost=${trace.cost ?? 'NA'})`, async () => { + await logs.openExplain(trace.id, 'cost'); + const text = await logs.readExplanation(); + expect(text.length).toBeGreaterThan(0); + expect(text).toMatch(KEYWORD_PATTERN.cost); + await logs.closeExplain(); + }); + } + + for (const trace of explainTraces) { + await test.step(`Explain the Duration cell for "${trace.name}" (duration=${trace.durationSeconds ?? 'NA'})`, async () => { + await logs.openExplain(trace.id, 'duration'); + const text = await logs.readExplanation(); + expect(text.length).toBeGreaterThan(0); + expect(text).toMatch(KEYWORD_PATTERN.duration); + await logs.closeExplain(); + }); + } + }); + + test('"Continue conversation" hands the same explanation off to the Ollie sidebar', async ({ + project, + explainTraces, + page, + }) => { + test.setTimeout(180_000); + const logs = new LogsPage(page); + const ollie = new OlliePage(page, project.id); + const trace = explainTraces[0]; + + await test.step('Open Logs and wait for the seeded traces to render', async () => { + await logs.goto(project.id); + await logs.waitForReady(); + }); + + await test.step('Open Explain on the Errors cell and read its settled text', async () => { + await logs.openExplain(trace.id, 'error'); + }); + + const explanation = await logs.readExplanation(); + + await test.step('Continue the conversation and confirm the sidebar shows the same answer', async () => { + const beforeCount = await ollie.messages().count(); + await logs.continueConversation(); + const sidebarText = await ollie.awaitContinuedConversation(beforeCount); + expect(normalize(sidebarText)).toContain(normalize(explanation)); + }); + }); +}); From d893679eddb85f65bd61cf9002c43d849747694d Mon Sep 17 00:00:00 2001 From: natagh23 Date: Wed, 22 Jul 2026 12:35:38 +0200 Subject: [PATCH 2/4] change tier of testing --- tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts b/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts index 326e9d192b0..e070258503c 100644 --- a/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts +++ b/tests_end_to_end/e2e/tests/ollie/ollie-explain.spec.ts @@ -32,7 +32,7 @@ const KEYWORD_PATTERN: Record<'cost' | 'duration', RegExp> = { // button label) that isn't part of the answer itself. const normalize = (text: string) => text.replace(/\s+/g, ' ').trim(); -test.describe('Ollie — explain cells', { tag: ['@t2-cuj', '@ollie'] }, () => { +test.describe('Ollie — explain cells', { tag: ['@t3-nightly', '@ollie'] }, () => { test.beforeEach(({ envConfig }) => skipIfOllieDisabled(envConfig)); test('Explain renders on-topic text for the Errors, Estimated cost, and Duration cell of every seeded trace', async ({ From fbf40593dd1cafa708c639fac4a72079f5a7a490 Mon Sep 17 00:00:00 2001 From: natagh23 Date: Wed, 22 Jul 2026 13:15:33 +0200 Subject: [PATCH 3/4] fix duplication --- tests_end_to_end/e2e/pom/ollie.page.ts | 78 ++++++++++---------------- 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/tests_end_to_end/e2e/pom/ollie.page.ts b/tests_end_to_end/e2e/pom/ollie.page.ts index eca7237fcac..89a13ca239b 100644 --- a/tests_end_to_end/e2e/pom/ollie.page.ts +++ b/tests_end_to_end/e2e/pom/ollie.page.ts @@ -68,24 +68,8 @@ export class OlliePage { await this.sendButton().click(); // The user echo and the assistant reply each mount as a [data-message-id] - // node. Wait for both to land (count grows by 2), then for the reply's - // text to be non-empty (it streams in after the bubble appears). - await expect - .poll(async () => this.messages().count(), { - timeout: timeoutMs, - intervals: [500, 1000, 2000], - }) - .toBeGreaterThanOrEqual(before + 2); - - const reply = this.messages().last(); - await expect - .poll(async () => ((await reply.textContent()) ?? '').trim().length, { - timeout: timeoutMs, - intervals: [500, 1000, 2000], - }) - .toBeGreaterThan(0); - - return ((await reply.textContent()) ?? '').trim(); + // node, so wait for both to land (count grows by 2). + return this.awaitNewMessage(before + 2, timeoutMs, [500, 1000, 2000]); }); } @@ -139,22 +123,7 @@ export class OlliePage { */ async awaitContinuedConversation(beforeCount: number, timeoutMs = 30_000): Promise { return test.step('wait for the continued conversation to render in the sidebar', async () => { - await expect - .poll(async () => this.messages().count(), { - timeout: timeoutMs, - intervals: [300, 600, 1200], - }) - .toBeGreaterThanOrEqual(beforeCount + 2); - - const reply = this.messages().last(); - await expect - .poll(async () => ((await reply.textContent()) ?? '').trim().length, { - timeout: timeoutMs, - intervals: [300, 600, 1200], - }) - .toBeGreaterThan(0); - - return ((await reply.textContent()) ?? '').trim(); + return this.awaitNewMessage(beforeCount + 2, timeoutMs, [300, 600, 1200]); }); } @@ -168,21 +137,7 @@ export class OlliePage { return test.step('run /analyze and await a response', async () => { const before = await this.messages().count(); await this.analyzeButton().click(); - await expect - .poll(async () => this.messages().count(), { - timeout: timeoutMs, - intervals: [1000, 2000, 5000], - }) - .toBeGreaterThan(before); - - const reply = this.messages().last(); - await expect - .poll(async () => ((await reply.textContent()) ?? '').trim().length, { - timeout: timeoutMs, - intervals: [1000, 2000, 5000], - }) - .toBeGreaterThan(0); - return ((await reply.textContent()) ?? '').trim(); + return this.awaitNewMessage(before + 1, timeoutMs, [1000, 2000, 5000]); }); } @@ -334,6 +289,31 @@ export class OlliePage { // ── private helpers ───────────────────────────────────────────────────── + /** + * Wait for the message list to reach `minCount` messages, then for the last + * one's text to be non-empty (it streams in after the bubble appears). + * Returns the last message's (the reply's) text. + */ + private async awaitNewMessage( + minCount: number, + timeoutMs: number, + intervals: number[], + ): Promise { + await expect + .poll(async () => this.messages().count(), { timeout: timeoutMs, intervals }) + .toBeGreaterThanOrEqual(minCount); + + const reply = this.messages().last(); + await expect + .poll(async () => ((await reply.textContent()) ?? '').trim().length, { + timeout: timeoutMs, + intervals, + }) + .toBeGreaterThan(0); + + return ((await reply.textContent()) ?? '').trim(); + } + // Anchor on the testid added to the host iframe in AssistantSidebar.tsx, OR // on the `title="Assistant"` attribute already present on deployed builds. // The `,` selector keeps the POM working against a cloud env running an older From 303896fa351b7298d315669dee7b7d86ac561d8f Mon Sep 17 00:00:00 2001 From: natagh23 Date: Wed, 22 Jul 2026 14:00:42 +0200 Subject: [PATCH 4/4] fix baz comments --- tests_end_to_end/e2e/pom/ollie.page.ts | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests_end_to_end/e2e/pom/ollie.page.ts b/tests_end_to_end/e2e/pom/ollie.page.ts index 89a13ca239b..29b3aeab213 100644 --- a/tests_end_to_end/e2e/pom/ollie.page.ts +++ b/tests_end_to_end/e2e/pom/ollie.page.ts @@ -299,19 +299,23 @@ export class OlliePage { timeoutMs: number, intervals: number[], ): Promise { - await expect - .poll(async () => this.messages().count(), { timeout: timeoutMs, intervals }) - .toBeGreaterThanOrEqual(minCount); - - const reply = this.messages().last(); - await expect - .poll(async () => ((await reply.textContent()) ?? '').trim().length, { - timeout: timeoutMs, - intervals, - }) - .toBeGreaterThan(0); - - return ((await reply.textContent()) ?? '').trim(); + return test.step('wait for a new Ollie message to render', async () => { + const deadline = Date.now() + timeoutMs; + + await expect + .poll(async () => this.messages().count(), { timeout: timeoutMs, intervals }) + .toBeGreaterThanOrEqual(minCount); + + const reply = this.messages().last(); + await expect + .poll(async () => ((await reply.textContent()) ?? '').trim().length, { + timeout: Math.max(0, deadline - Date.now()), + intervals, + }) + .toBeGreaterThan(0); + + return ((await reply.textContent()) ?? '').trim(); + }); } // Anchor on the testid added to the host iframe in AssistantSidebar.tsx, OR