|
| 1 | +import { randomUUID } from "node:crypto"; |
| 2 | +import fs from "node:fs/promises"; |
| 3 | +import os from "node:os"; |
| 4 | +import path from "node:path"; |
| 5 | +import type { |
| 6 | + CodexBundleMcpThreadConfig, |
| 7 | + EmbeddedRunAttemptParamsV2 as EmbeddedRunAttemptParams, |
| 8 | +} from "openclaw/plugin-sdk/agent-harness-runtime"; |
| 9 | +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; |
| 10 | +import { startCodexAttemptThread } from "./attempt-startup.js"; |
| 11 | +import { threadStartResult } from "./codex-app-server.test-fixtures.js"; |
| 12 | +import { |
| 13 | + resolveCodexAppServerRuntimeOptions, |
| 14 | + resolveCodexComputerUseConfig, |
| 15 | + type CodexPluginConfig, |
| 16 | +} from "./config.js"; |
| 17 | +import { createCodexTestHostCapabilities } from "./host-capability.test-support.js"; |
| 18 | +import { defaultCodexPluginMetadataCache } from "./plugin-metadata-cache.js"; |
| 19 | +import { |
| 20 | + resetCodexTestBindingStore, |
| 21 | + testCodexAppServerBindingStore, |
| 22 | +} from "./session-binding.test-helpers.js"; |
| 23 | +import { |
| 24 | + clearSharedCodexAppServerClient, |
| 25 | + clearSharedCodexAppServerClientAndWait, |
| 26 | + getLeasedSharedCodexAppServerClient, |
| 27 | +} from "./shared-client.js"; |
| 28 | +import { createCodexTestModel } from "./test-support.js"; |
| 29 | + |
| 30 | +vi.mock("./desktop-generation.js", () => ({ |
| 31 | + isCodexDesktopGenerationCurrent: () => false, |
| 32 | + waitForCodexDesktopGeneration: async () => undefined, |
| 33 | +})); |
| 34 | + |
| 35 | +const tempRoots = new Set<string>(); |
| 36 | + |
| 37 | +async function createStartupFailureFixture( |
| 38 | + mode: "transient" | "contention" | "persistent" | "unsupported", |
| 39 | +) { |
| 40 | + const root = path.join(os.tmpdir(), `openclaw-codex-startup-retry-${randomUUID()}`); |
| 41 | + tempRoots.add(root); |
| 42 | + const fixturePath = path.join(root, "startup-failure.mjs"); |
| 43 | + const spawnCountPath = path.join(root, "spawn-count"); |
| 44 | + const codexHome = path.join(root, "codex-home"); |
| 45 | + await fs.mkdir(root, { recursive: true }); |
| 46 | + await fs.writeFile( |
| 47 | + fixturePath, |
| 48 | + [ |
| 49 | + 'import fs from "node:fs";', |
| 50 | + 'import readline from "node:readline";', |
| 51 | + "const [spawnCountPath, mode, codexHome] = process.argv.slice(2);", |
| 52 | + 'const attempt = Number(fs.existsSync(spawnCountPath) ? fs.readFileSync(spawnCountPath, "utf8") : 0) + 1;', |
| 53 | + 'fs.writeFileSync(spawnCountPath, String(attempt), "utf8");', |
| 54 | + "const startedAtPath = `${spawnCountPath}.started-at`;", |
| 55 | + 'if (attempt === 1) fs.writeFileSync(startedAtPath, String(Date.now()), "utf8");', |
| 56 | + 'const stillContended = mode === "contention" && Date.now() - Number(fs.readFileSync(startedAtPath, "utf8")) < 750;', |
| 57 | + 'if (mode === "persistent" || (mode === "transient" && attempt === 1) || stillContended) {', |
| 58 | + " console.error(`Error: failed to initialize sqlite state runtime under ${codexHome}: failed to initialize state runtime at ${codexHome}`);", |
| 59 | + " process.exitCode = 1;", |
| 60 | + "} else {", |
| 61 | + " const lines = readline.createInterface({ input: process.stdin });", |
| 62 | + ' lines.on("line", (line) => {', |
| 63 | + " const message = JSON.parse(line);", |
| 64 | + " if (message.id === undefined) return;", |
| 65 | + ' const result = message.method === "initialize"', |
| 66 | + ' ? { userAgent: `openclaw/${mode === "unsupported" ? "0.1.0" : "0.149.0"} (macOS; test)` }', |
| 67 | + ` : ${JSON.stringify(threadStartResult("thread-recovered", "/repo"))};`, |
| 68 | + " process.stdout.write(`${JSON.stringify({ id: message.id, result })}\\n`);", |
| 69 | + " });", |
| 70 | + "}", |
| 71 | + ].join("\n"), |
| 72 | + "utf8", |
| 73 | + ); |
| 74 | + const pluginConfig = { |
| 75 | + appServer: { |
| 76 | + transport: "stdio", |
| 77 | + command: process.execPath, |
| 78 | + args: [fixturePath, spawnCountPath, mode, codexHome], |
| 79 | + requestTimeoutMs: 5_000, |
| 80 | + }, |
| 81 | + } satisfies CodexPluginConfig; |
| 82 | + return { root, spawnCountPath, pluginConfig }; |
| 83 | +} |
| 84 | + |
| 85 | +function startFixtureAttempt(fixture: Awaited<ReturnType<typeof createStartupFailureFixture>>) { |
| 86 | + const agentDir = path.join(fixture.root, "agent"); |
| 87 | + const workspaceDir = path.join(fixture.root, "workspace"); |
| 88 | + const bundleMcpThreadConfig = { |
| 89 | + configPatch: undefined, |
| 90 | + diagnostics: [], |
| 91 | + evaluated: false, |
| 92 | + fingerprint: undefined, |
| 93 | + staticServerNames: [], |
| 94 | + userStaticServerNames: [], |
| 95 | + } satisfies CodexBundleMcpThreadConfig; |
| 96 | + return startCodexAttemptThread({ |
| 97 | + bindingStore: testCodexAppServerBindingStore, |
| 98 | + attemptClientFactory: getLeasedSharedCodexAppServerClient, |
| 99 | + appServer: resolveCodexAppServerRuntimeOptions({ pluginConfig: fixture.pluginConfig }), |
| 100 | + pluginConfig: fixture.pluginConfig, |
| 101 | + computerUseConfig: resolveCodexComputerUseConfig({ pluginConfig: fixture.pluginConfig }), |
| 102 | + startupAuthProfileId: undefined, |
| 103 | + startupAuthBindingFingerprint: undefined, |
| 104 | + startupAuthAccountCacheKey: undefined, |
| 105 | + startupEnvApiKeyCacheKey: undefined, |
| 106 | + agentDir, |
| 107 | + config: undefined, |
| 108 | + buildAttemptParams: () => |
| 109 | + ({ |
| 110 | + hostCapabilities: createCodexTestHostCapabilities(), |
| 111 | + prompt: "hello", |
| 112 | + sessionId: "session-1", |
| 113 | + sessionKey: "agent:agent-1:session-1", |
| 114 | + agentDir, |
| 115 | + sessionFile: path.join(fixture.root, "session.jsonl"), |
| 116 | + effectiveCwd: workspaceDir, |
| 117 | + workspaceDir, |
| 118 | + runId: "run-1", |
| 119 | + provider: "codex", |
| 120 | + modelId: "gpt-5.4-codex", |
| 121 | + model: createCodexTestModel("codex"), |
| 122 | + thinkLevel: "medium", |
| 123 | + disableTools: true, |
| 124 | + timeoutMs: 5_000, |
| 125 | + authStorage: {} as never, |
| 126 | + authProfileStore: { version: 1, profiles: {} }, |
| 127 | + modelRegistry: {} as never, |
| 128 | + }) as EmbeddedRunAttemptParams, |
| 129 | + sessionAgentId: "agent-1", |
| 130 | + effectiveWorkspace: workspaceDir, |
| 131 | + effectiveCwd: workspaceDir, |
| 132 | + dynamicTools: [], |
| 133 | + webSearchAllowed: false, |
| 134 | + developerInstructions: undefined, |
| 135 | + finalConfigPatch: undefined, |
| 136 | + bundleMcpThreadConfig, |
| 137 | + nativeToolSurfaceEnabled: true, |
| 138 | + nativeProviderWebSearchSupport: "supported", |
| 139 | + sandboxExecServerEnabled: false, |
| 140 | + sandbox: null, |
| 141 | + contextEngineProjection: undefined, |
| 142 | + startupTimeoutMs: 10_000, |
| 143 | + signal: new AbortController().signal, |
| 144 | + onStartupTimeout: vi.fn(), |
| 145 | + spawnedBy: undefined, |
| 146 | + }); |
| 147 | +} |
| 148 | + |
| 149 | +describe("Codex app-server startup retry", () => { |
| 150 | + beforeEach(() => { |
| 151 | + vi.stubEnv("CODEX_API_KEY", ""); |
| 152 | + vi.stubEnv("OPENAI_API_KEY", ""); |
| 153 | + clearSharedCodexAppServerClient(); |
| 154 | + defaultCodexPluginMetadataCache.clear(); |
| 155 | + resetCodexTestBindingStore(); |
| 156 | + }); |
| 157 | + |
| 158 | + afterEach(async () => { |
| 159 | + await clearSharedCodexAppServerClientAndWait(); |
| 160 | + defaultCodexPluginMetadataCache.clear(); |
| 161 | + vi.unstubAllEnvs(); |
| 162 | + for (const root of tempRoots) { |
| 163 | + await fs.rm(root, { recursive: true, force: true }); |
| 164 | + } |
| 165 | + tempRoots.clear(); |
| 166 | + }); |
| 167 | + |
| 168 | + it("retries a real app-server after transient sqlite state initialization failure", async () => { |
| 169 | + const fixture = await createStartupFailureFixture("transient"); |
| 170 | + const result = await startFixtureAttempt(fixture); |
| 171 | + |
| 172 | + expect(result.thread.threadId).toBe("thread-recovered"); |
| 173 | + expect(await fs.readFile(fixture.spawnCountPath, "utf8")).toBe("2"); |
| 174 | + result.turnRoute.release(); |
| 175 | + result.releaseSharedClientLease(); |
| 176 | + }); |
| 177 | + |
| 178 | + it("waits out transient sqlite contention before retrying app-server startup", async () => { |
| 179 | + const fixture = await createStartupFailureFixture("contention"); |
| 180 | + const result = await startFixtureAttempt(fixture); |
| 181 | + |
| 182 | + expect(result.thread.threadId).toBe("thread-recovered"); |
| 183 | + expect(await fs.readFile(fixture.spawnCountPath, "utf8")).toBe("2"); |
| 184 | + result.turnRoute.release(); |
| 185 | + result.releaseSharedClientLease(); |
| 186 | + }); |
| 187 | + |
| 188 | + it("bounds retries when sqlite state initialization keeps failing", async () => { |
| 189 | + const fixture = await createStartupFailureFixture("persistent"); |
| 190 | + |
| 191 | + await expect(startFixtureAttempt(fixture)).rejects.toThrow( |
| 192 | + "failed to initialize sqlite state runtime", |
| 193 | + ); |
| 194 | + expect(await fs.readFile(fixture.spawnCountPath, "utf8")).toBe("3"); |
| 195 | + }); |
| 196 | + |
| 197 | + it("rejects an unsupported app-server version without retrying", async () => { |
| 198 | + const fixture = await createStartupFailureFixture("unsupported"); |
| 199 | + |
| 200 | + await expect(startFixtureAttempt(fixture)).rejects.toThrow( |
| 201 | + /app-server .* or newer is required/i, |
| 202 | + ); |
| 203 | + expect(await fs.readFile(fixture.spawnCountPath, "utf8")).toBe("1"); |
| 204 | + }); |
| 205 | +}); |
0 commit comments