test(e2e): Add Mastra Jev classifier test to node-mastra (#25115)

## What

Adds a node-mastra e2e test that runs a real Jev evaluation through a
Mastra `Classifier` (via OpenRouter) and checks the `gen_ai.evaluate`
span.

## Why

Covers the classifier instrumentation from #25044 against the real
provider, not only a mock model.

Author: andreiborza

dev-packages/e2e-tests/test-applications/node-mastra/package.json

@@ -15,11 +15,11 @@
     "test:dev": "TEST_ENV=development playwright test"
   },
   "dependencies": {
-    "@mastra/core": "~1.65.0",
+    "@mastra/core": "~1.74.0",
     "@mastra/libsql": "~1.22.4",
     "@mastra/memory": "~1.28.3",
     "@mastra/observability": "~1.17.6",
-    "@openrouter/ai-sdk-provider": "~3.0.0",
+    "@openrouter/ai-sdk-provider": "~3.1.0",
     "@sentry/node": "file:../../packed/sentry-node-packed.tgz",
     "ai": "~7.0.97",
     "dataloader": "~2.2.3",
@@ -28,6 +28,7 @@
   "devDependencies": {
     "@playwright/test": "~1.56.0",
     "@sentry-internal/test-utils": "link:../../../test-utils",
+    "@sentry/conventions": "0.26.0",
     "@types/node": "24.x",
     "mastra": "~1.28.0",
     "typescript": "~5.9.0"

dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/agents/weather-agent.ts

@@ -1,19 +1,10 @@
-import { createOpenRouter } from '@openrouter/ai-sdk-provider';
 import { Agent } from '@mastra/core/agent';
 import { Memory } from '@mastra/memory';
 import { LibSQLStore } from '@mastra/libsql';
 import { countItemsTool } from '../tools/count-items.js';
 import { failNowTool } from '../tools/fail-now.js';
 import { getWeatherTool } from '../tools/get-weather.js';
-
-const apiKey = process.env.E2E_OPENROUTER_API_KEY;
-if (!apiKey) {
-  throw new Error('E2E_OPENROUTER_API_KEY is not set');
-}
-
-// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the
-// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`.
-const openrouter = createOpenRouter({ apiKey });
+import { openrouter } from '../openrouter.js';
 
 // An in-memory libsql store is enough for the test: Mastra requires a storage
 // provider before `generate(..., { memory: { thread, resource } })` is accepted,

dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/classifiers/ticket-classifier.ts

@@ -0,0 +1,17 @@
+import { Classifier } from '@mastra/core/classifier';
+import { openrouter } from '../openrouter.js';
+
+export const TICKET_CLASSIFIER = 'ticketClassifier';
+
+// A Jev evaluation through OpenRouter's Decisions API. `Classifier` calls the model's `doEvaluate()`
+// directly, so its span comes from the Mastra exporter, not from the Vercel AI integration.
+export const ticketClassifier = new Classifier({
+  id: TICKET_CLASSIFIER,
+  model: openrouter.evaluationModel('typesafe/jev-1.13'),
+  questions: {
+    is_bug: {
+      type: 'boolean',
+      instructions: 'Is the customer reporting a software defect?',
+    },
+  },
+});

dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/index.ts

@@ -2,6 +2,7 @@ import { Mastra } from '@mastra/core';
 import { registerApiRoute } from '@mastra/core/server';
 import { LibSQLStore } from '@mastra/libsql';
 import { WEATHER_AGENT, weatherAgent } from './agents/weather-agent.js';
+import { TICKET_CLASSIFIER, ticketClassifier } from './classifiers/ticket-classifier.js';
 
 // The agent is driven through Mastra's built-in `POST /api/agents/:id/generate`
 // endpoint (see tests/utils.ts). `dataloader` (orchestrion-instrumented) is exercised
@@ -19,12 +20,24 @@ const manualRoute = registerApiRoute('/manual-route', {
   },
 });
 
+// Evaluates the registered classifier outside of any agent run. Registration lets Mastra start a root
+// `classifier_evaluation` span when no Mastra span is active.
+const classifyRoute = registerApiRoute('/classify', {
+  method: 'POST',
+  async handler(c) {
+    const { ticket } = await c.req.json<{ ticket: string }>();
+    const result = await ticketClassifier.evaluate({ state: { ticket } });
+    return c.json({ answers: result.answers });
+  },
+});
+
 export const mastra = new Mastra({
   agents: { [WEATHER_AGENT]: weatherAgent },
+  classifiers: { [TICKET_CLASSIFIER]: ticketClassifier },
   storage: new LibSQLStore({ id: 'mastra-storage', url: ':memory:' }),
   server: {
     port: 4111,
-    apiRoutes: [manualRoute],
+    apiRoutes: [manualRoute, classifyRoute],
   },
   // No `bundler.externals` override: `mastra build` defaults to externalizing all non-workspace deps,
   // so Hono (and the other instrumented modules) stay unbundled and the `--import` orchestrion hook

dev-packages/e2e-tests/test-applications/node-mastra/src/mastra/openrouter.ts

@@ -0,0 +1,10 @@
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+
+const apiKey = process.env.E2E_OPENROUTER_API_KEY;
+if (!apiKey) {
+  throw new Error('E2E_OPENROUTER_API_KEY is not set');
+}
+
+// Call OpenRouter directly (rather than the default Vercel AI Gateway) so the
+// e2e test needs only a single OpenRouter key, reusing `E2E_OPENROUTER_API_KEY`.
+export const openrouter = createOpenRouter({ apiKey });

dev-packages/e2e-tests/test-applications/node-mastra/tests/classifier.test.ts

@@ -0,0 +1,65 @@
+import { expect, test } from '@playwright/test';
+import {
+  GEN_AI_INPUT_MESSAGES,
+  GEN_AI_OPERATION_NAME,
+  GEN_AI_OUTPUT_MESSAGES,
+  GEN_AI_PROVIDER_NAME,
+  GEN_AI_REQUEST_MODEL,
+  GEN_AI_USAGE_INPUT_TOKENS,
+  SENTRY_ORIGIN,
+  URL_FULL,
+} from '@sentry/conventions/attributes';
+import { GEN_AI_EVALUATE, HTTP_SERVER } from '@sentry/conventions/op';
+import { collectStreamedSpans, getSpanOp, SerializedStreamedSpan } from '@sentry-internal/test-utils';
+
+const APP = 'node-mastra';
+const TICKET = 'My checkout page shows a blank screen after I click Pay.';
+
+const isEvaluateSpan = (span: SerializedStreamedSpan): boolean => getSpanOp(span) === GEN_AI_EVALUATE;
+const isClassifyServerSpan = (span: SerializedStreamedSpan): boolean =>
+  getSpanOp(span) === HTTP_SERVER && String(span.attributes?.[URL_FULL]?.value ?? '').includes('/classify');
+
+test('captures a Mastra classifier (Jev) evaluation as a gen_ai.evaluate span', async ({ baseURL }) => {
+  const spansPromise = collectStreamedSpans(
+    APP,
+    spansOfTrace => spansOfTrace.some(isEvaluateSpan) && spansOfTrace.some(isClassifyServerSpan),
+  );
+
+  const res = await fetch(`${baseURL}/classify`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ ticket: TICKET }),
+  });
+  expect(res.status).toBe(200);
+  await res.json();
+
+  const spans = await spansPromise;
+  const serverSpan = spans.find(isClassifyServerSpan)!;
+
+  // Only the Mastra exporter reports the evaluation: `Classifier` calls `doEvaluate()` directly, so the
+  // Vercel AI integration does not see it.
+  const evaluateSpans = spans.filter(isEvaluateSpan);
+  expect(evaluateSpans.map(span => ({ name: span.name, origin: span.attributes?.[SENTRY_ORIGIN]?.value }))).toEqual([
+    { name: 'evaluate typesafe/jev-1.13', origin: 'auto.ai.mastra' },
+  ]);
+
+  const evaluateSpan = evaluateSpans[0]!;
+  expect(evaluateSpan.parent_span_id).toBe(serverSpan.span_id);
+  expect(evaluateSpan.attributes?.[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate');
+  expect(evaluateSpan.attributes?.[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe/jev-1.13');
+  expect(evaluateSpan.attributes?.[GEN_AI_PROVIDER_NAME]?.value).toBe('openrouter');
+  expect(evaluateSpan.attributes?.[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBeGreaterThan(0);
+  expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_INPUT_MESSAGES]?.value))).toEqual([
+    {
+      type: 'evaluation',
+      state: { ticket: TICKET },
+      questions: {
+        is_bug: { type: 'boolean', instructions: 'Is the customer reporting a software defect?' },
+      },
+    },
+  ]);
+  // The probability comes from the live model, so only its shape is stable.
+  expect(JSON.parse(String(evaluateSpan.attributes?.[GEN_AI_OUTPUT_MESSAGES]?.value))).toEqual([
+    { type: 'evaluation', answers: { is_bug: { type: 'boolean', probability: expect.any(Number) } } },
+  ]);
+});