Skip to content

Commit e830d73

Browse files
author
backryun
committed
test(tiers): split slow e2e and upstream-gated live out of default integration glob
Moves 8 slow hermetic suites (batch-e2e 150s, live-ws-heartbeat 54s, resilience-http 36s, +5) to tests/integration/e2e/ with a dedicated test:integration:e2e script, and 7 upstream-gated suites (live-gemini*, gemini-live-429, codex-chat-reasoning, wire-capture) to tests/integration/live/ with test:integration:live. Shared helpers used by both tiers stay at tests/integration/ root; tier-private helpers move along. CI runs both tiers after the default suite so no assertions stop running anywhere. Default test:integration: 115 -> 100 files. Pre-existing failures (combo-failover/routing assertions, resilience-http-e2e 1 case) unchanged and out of scope. Refs #12539
1 parent f88979f commit e830d73

21 files changed

Lines changed: 51 additions & 36 deletions

‎.github/workflows/ci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1339,6 +1339,13 @@ jobs:
13391339
env:
13401340
TEST_SHARD: ${{ matrix.shard }}/2
13411341
run: npm run test:integration:ci
1342+
# #12539 tier-split: slow hermetic e2e lives in tests/integration/e2e/ so the
1343+
# default glob stays fast; run it here so the assertions still execute in CI.
1344+
- name: Integration e2e tier (slow hermetic)
1345+
run: npm run test:integration:e2e
1346+
# Upstream-gated live tier: self-skips without OMNIROUTE_API_KEY, ~seconds.
1347+
- name: Integration live tier (self-skipping without creds)
1348+
run: npm run test:integration:live
13421349

13431350
test-security:
13441351
name: Security Tests

‎config/quality/eslint-suppressions.json‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2287,7 +2287,7 @@
22872287
"count": 19
22882288
}
22892289
},
2290-
"tests/integration/combo-routing-e2e.test.ts": {
2290+
"tests/integration/e2e/combo-routing-e2e.test.ts": {
22912291
"@typescript-eslint/no-explicit-any": {
22922292
"count": 7
22932293
}
@@ -2307,28 +2307,28 @@
23072307
"count": 7
23082308
}
23092309
},
2310-
"tests/integration/live-gemini-agentic-loop.test.ts": {
2310+
"tests/integration/live/live-gemini-agentic-loop.test.ts": {
23112311
"@typescript-eslint/no-unused-vars": {
23122312
"count": 2
23132313
}
23142314
},
2315-
"tests/integration/live-gemini-nonstream.test.ts": {
2315+
"tests/integration/live/live-gemini-nonstream.test.ts": {
23162316
"@typescript-eslint/no-explicit-any": {
23172317
"count": 1
23182318
},
23192319
"@typescript-eslint/no-unused-vars": {
23202320
"count": 1
23212321
}
23222322
},
2323-
"tests/integration/live-gemini-workload.test.ts": {
2323+
"tests/integration/live/live-gemini-workload.test.ts": {
23242324
"@typescript-eslint/no-explicit-any": {
23252325
"count": 1
23262326
},
23272327
"@typescript-eslint/no-unused-vars": {
23282328
"count": 1
23292329
}
23302330
},
2331-
"tests/integration/live-gemini.test.ts": {
2331+
"tests/integration/live/live-gemini.test.ts": {
23322332
"@typescript-eslint/no-unused-vars": {
23332333
"count": 2
23342334
}
@@ -2421,7 +2421,7 @@
24212421
"count": 1
24222422
}
24232423
},
2424-
"tests/integration/resilience-http-e2e.test.ts": {
2424+
"tests/integration/e2e/resilience-http-e2e.test.ts": {
24252425
"@typescript-eslint/no-explicit-any": {
24262426
"count": 3
24272427
}

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,9 @@
278278
"release:uncovered": "node scripts/release/list-uncovered-commits.mjs",
279279
"test:coverage:runner": "node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=8 tests/unit/*.test.ts \"tests/unit/{a2a,account,adaptive,admission,adobe,agent,agentSkills,agentrouter,agy,aihorde,alibaba,anthropic,antigravity,api,apikeys,audio,audit,auth,authz,auto,bailian,base,batch,blackbox,bug,build,bulk,cache,call,catalog,cc,chat,chatcore,chatgpt,check,circuit,claude,cli,cli-helper,cline,cliproxyapi,cloud,cloudflare,codex,combo,combos,command,compression,conductor,connection,context,copilot,correctness,cors,credential,crof,cursor,custom,dahl,db,db-adapters,deepseek,dev,devin,docker,dockerfile,docs,domain,duckduckgo,electron,embedding,embeddings,empty,error,exclusive,executor,firecrawl,fix,free,freeProviderRankings,fusion,gamification,gemini,github,gitlab,glm,grok,guardrails,headroom,helpers,home,homolog,i18n,image,inspector,instrumentation,issue,json,kie,kimi,kiro,lib,live,lmarena,local,log,m365,management,mcp,media,memory,microsoft,migration,minimax,misc,mitm,modality,model,models,moonshot,muse,no,noauth,notion,nvidia,oauth,obsidian,ocr,ollama,openai,openapi,opencode,openrouter,pack,perplexity,playground,plugins,pricing,probe,prompt,provider,providers,proxy,proxyfetch,qoder,quota,qwen,radar,rate,rateLimitManager,reasoning,refactor,relay,remote,repro,request,rerank,resilience,resolve,resource,responses,route,router,routing,run,runtime,search,security,seekai,service,services,session,settings,shared,sidebar,skills,socks,sse,stream,sync,synced,system,task,test,thinking,tls,token,tool,topology,tproxy,translator,ts7,uc,ui,upstream,usage,v1,v388,validation,vercel,vertex,video,vision,vscode,web,webhook,windows,xai,zai,zed}/**/*.test.ts\" \"tests/unit/**/*.test.mjs\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true NODE_OPTIONS=--max-old-space-size=8192 c8 --merge-async --output-dir coverage --exclude=tests/** --exclude=**/*.test.* --reporter=text-summary --reporter=html --reporter=json-summary --reporter=lcov --check-coverage --statements 60 --lines 60 --functions 60 --branches 60 node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=8 \"tests/unit/dashboard/**/*.test.ts\" && npm run test:unit:serial",
280280
"test:unit:serial": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/unit/serial/**/*.test.ts\"",
281-
"alibaba:sync-allowlist": "node --import tsx/esm scripts/ops/sync-alibaba-allowlist.mjs"
281+
"alibaba:sync-allowlist": "node --import tsx/esm scripts/ops/sync-alibaba-allowlist.mjs",
282+
"test:integration:e2e": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/integration/e2e/*.test.ts\"",
283+
"test:integration:live": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/integration/live/*.test.ts\""
282284
},
283285
"dependencies": {
284286
"@aws-sdk/client-bedrock-runtime": "^3.1120.0",

‎scripts/check/check-test-discovery.mjs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,12 @@ export const COLLECTORS = [
9292
{ glob: "tests/integration/combo-matrix/*.test.ts", sources: ["package.json"] },
9393
// Node native runner — test:combo:live (gated real-upstream smoke; RUN_COMBO_LIVE=1 + VPS creds)
9494
{ glob: "tests/integration/combo-live/*.live.test.ts", sources: ["package.json"] },
95+
// Node native runner — test:integration:e2e (slow hermetic e2e: playwright spawn,
96+
// 30s+ startups; out of the default test:integration glob since #12539 tier-split)
97+
{ glob: "tests/integration/e2e/*.test.ts", sources: ["package.json"] },
98+
// Node native runner — test:integration:live (upstream-gated; self-skips without
99+
// OMNIROUTE_API_KEY / RUN_LIVE_WIRE_CAPTURE; out of the default glob since #12539)
100+
{ glob: "tests/integration/live/*.test.ts", sources: ["package.json"] },
95101
// Node native runner — test:boundary:live (gated real-upstream smoke; RUN_BOUNDARY_LIVE=1,
96102
// hits omniroute.vhost2.harre.dynv6.net — never runs unopted in CI)
97103
{ glob: "tests/boundary/*.live.test.ts", sources: ["package.json"] },

tests/integration/batch-e2e-rate-limit.test.ts renamed to tests/integration/e2e/batch-e2e-rate-limit.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { spawn } from "node:child_process";
99
import { fileURLToPath } from "node:url";
1010

1111
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-batch-e2e-rl-"));
12-
const REPO_ROOT = fileURLToPath(new URL("../..", import.meta.url));
12+
const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url));
1313
const RELAY_PORT = await getFreePort();
1414
const SERVER_PORT = await getFreePort();
1515

tests/integration/combo-failover-e2e.test.ts renamed to tests/integration/e2e/combo-failover-e2e.test.ts

File renamed without changes.

tests/integration/combo-routing-e2e.test.ts renamed to tests/integration/e2e/combo-routing-e2e.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import assert from "node:assert/strict";
44
import { createChatPipelineHarness } from "./_chatPipelineHarness.ts";
55

66
const harness = await createChatPipelineHarness("combo-routing");
7-
const callLogs = await import("../../src/lib/usage/callLogs.ts");
7+
const callLogs = await import("../../../src/lib/usage/callLogs.ts");
88
const {
99
BaseExecutor,
1010
buildClaudeResponse,

tests/integration/cursor-e2e.test.ts renamed to tests/integration/e2e/cursor-e2e.test.ts

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ test(
9999
"[cursor-e2e] single-turn plain chat returns assistant text",
100100
{ skip: skipReason },
101101
async () => {
102-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
102+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
103103
const exec = new CursorExecutor();
104104
const result = await exec.execute({
105105
model: "auto",
@@ -118,7 +118,7 @@ test(
118118
);
119119

120120
test("[cursor-e2e] system prompt biases the response", { skip: skipReason }, async () => {
121-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
121+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
122122
const exec = new CursorExecutor();
123123
const result = await exec.execute({
124124
model: "auto",
@@ -140,7 +140,7 @@ test("[cursor-e2e] system prompt biases the response", { skip: skipReason }, asy
140140
});
141141

142142
test("[cursor-e2e] tool-use single-turn returns tool_calls", { skip: skipReason }, async () => {
143-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
143+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
144144
const exec = new CursorExecutor();
145145
const result = await exec.execute({
146146
model: "claude-4.6-sonnet-medium",
@@ -180,7 +180,7 @@ test(
180180
"[cursor-e2e] streaming SSE delivers chunks before the upstream closes",
181181
{ skip: skipReason },
182182
async () => {
183-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
183+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
184184
const exec = new CursorExecutor();
185185
const result = await exec.execute({
186186
model: "auto",
@@ -224,7 +224,7 @@ test(
224224
// fallback. All were validated end-to-end against the live endpoint.
225225

226226
test("[cursor-e2e] composer-2.5 plain chat returns assistant text", { skip: skipReason }, async () => {
227-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
227+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
228228
const exec = new CursorExecutor();
229229
const result = await exec.execute({
230230
model: COMPOSER_MODEL,
@@ -247,7 +247,7 @@ test(
247247
"[cursor-e2e] composer-2.5 surfaces reasoning as reasoning_content",
248248
{ skip: skipReason },
249249
async () => {
250-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
250+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
251251
const exec = new CursorExecutor();
252252
const result = await exec.execute({
253253
model: COMPOSER_MODEL,
@@ -276,9 +276,9 @@ test(
276276
"[cursor-e2e] composer-2.5 multi-turn tool round-trip reuses the h2 session",
277277
{ skip: skipReason },
278278
async () => {
279-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
279+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
280280
const { cursorSessionManager } = await import(
281-
"../../open-sse/services/cursorSessionManager.ts"
281+
"../../../open-sse/services/cursorSessionManager.ts"
282282
);
283283
const exec = new CursorExecutor();
284284
const conversationId = `e2e-rt-${Date.now()}`;
@@ -340,7 +340,7 @@ test(
340340
"[cursor-e2e] composer-2.5 cold-resume incorporates a tool result without a live session",
341341
{ skip: skipReason },
342342
async () => {
343-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
343+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
344344
const exec = new CursorExecutor();
345345
// Brand-new conversation_id with a fabricated prior tool call/result and no
346346
// session ever opened → acquire() misses, exercising the cold-resume path
@@ -387,7 +387,7 @@ test(
387387
"[cursor-e2e] composer-2.5 honors response_format json_object",
388388
{ skip: skipReason },
389389
async () => {
390-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
390+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
391391
const exec = new CursorExecutor();
392392
const result = await exec.execute({
393393
model: COMPOSER_MODEL,
@@ -417,7 +417,7 @@ test(
417417
"[cursor-e2e] composer-2.5 streaming delivers incremental chunks",
418418
{ skip: skipReason },
419419
async () => {
420-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
420+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
421421
const exec = new CursorExecutor();
422422
const result = await exec.execute({
423423
model: COMPOSER_MODEL,
@@ -452,7 +452,7 @@ test(
452452
"[cursor-e2e] base64 image_url reaches a vision model (sees the color)",
453453
{ skip: skipReason },
454454
async () => {
455-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
455+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
456456
const exec = new CursorExecutor();
457457
const png = solidColorPng(64, [255, 0, 0]); // solid red
458458
const dataUri = `data:image/png;base64,${png.toString("base64")}`;
@@ -493,7 +493,7 @@ test(
493493
"[cursor-e2e] remote image_url is fetched and reaches a vision model",
494494
{ skip: skipReason },
495495
async () => {
496-
const { CursorExecutor } = await import("../../open-sse/executors/cursor.ts");
496+
const { CursorExecutor } = await import("../../../open-sse/executors/cursor.ts");
497497
const exec = new CursorExecutor();
498498
const result = await exec.execute({
499499
model: VISION_MODEL,

tests/integration/live-default-combo-workload.test.ts renamed to tests/integration/e2e/live-default-combo-workload.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import {
1818
getDefaultComboModelTargets,
1919
filterActiveModelTargets,
2020
sendModelRequest,
21-
} from "./liveDefaultComboShared.ts";
21+
} from "../liveDefaultComboShared.ts";
2222

2323
let modelNames: string[] = [];
2424

tests/integration/live-ws-heartbeat-keepalive.test.ts renamed to tests/integration/e2e/live-ws-heartbeat-keepalive.test.ts

File renamed without changes.

0 commit comments

Comments
 (0)