diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts index bcd5b20a..53a3221c 100644 --- a/src/sdk-runner/index.ts +++ b/src/sdk-runner/index.ts @@ -36,6 +36,45 @@ function selfInvoke(args: string[], cwd: string): Promise { }); } +// Connect errors that mean nothing is listening at the address. +const NOBODY_LISTENING = new Set([ + 'ECONNREFUSED', + 'EADDRNOTAVAIL', + 'ENETUNREACH', + 'EHOSTUNREACH' +]); + +function nobodyListening(err: unknown): boolean { + const cause = (err as { cause?: { code?: string; errors?: unknown[] } }) + ?.cause; + if (!cause) return false; + // A localhost url tries ::1 and 127.0.0.1, and fetch reports both errors. + if (Array.isArray(cause.errors) && cause.errors.length > 0) { + return cause.errors.every((e) => + NOBODY_LISTENING.has((e as { code?: string })?.code ?? '') + ); + } + return NOBODY_LISTENING.has(cause.code ?? ''); +} + +// Returns why the url looks held, or null when nothing is listening there. +// Only a refused connect counts as free. A process that accepts the socket +// and then stalls or hangs up still holds the port. +async function heldBy(url: string, timeoutMs: number): Promise { + try { + await fetch(url, { method: 'GET', signal: AbortSignal.timeout(timeoutMs) }); + return 'something answered'; + } catch (err) { + if (nobodyListening(err)) return null; + if ((err as Error)?.name === 'TimeoutError') { + return `something accepted the connection but did not answer within ${timeoutMs}ms`; + } + const cause = (err as { cause?: { code?: string; message?: string } }) + ?.cause; + return `the probe failed with ${cause?.code ?? cause?.message ?? String(err)}`; + } +} + async function waitForReady(url: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; // Per-probe timeout: a server that accepts the socket but never responds must @@ -66,6 +105,16 @@ export async function withManagedServer( readyTimeoutMs: number, fn: () => Promise ): Promise { + // A leftover process on the same port would pass the readiness probe, and + // the run would grade it instead of the server we are about to start. + const held = await heldBy(url, 1000); + if (held) { + throw new Error( + `${url} is already in use before the server started (${held}). ` + + `Stop that process or change the port, or this run would grade it ` + + `instead of the SDK's server.` + ); + } console.error(`[sdk] Starting server: ${command}`); const child: ChildProcess = spawn(command, { shell: true, diff --git a/src/sdk-runner/managed-server.test.ts b/src/sdk-runner/managed-server.test.ts new file mode 100644 index 00000000..37f64ef5 --- /dev/null +++ b/src/sdk-runner/managed-server.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest'; +import http from 'http'; +import net from 'net'; +import type { AddressInfo } from 'net'; +import { withManagedServer } from './index'; + +function listen(server: http.Server, port = 0): Promise { + return new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => + resolve((server.address() as AddressInfo).port) + ); + }); +} + +async function freePort(): Promise { + const s = http.createServer(); + const port = await listen(s); + await new Promise((r) => s.close(r)); + return port; +} + +function serverCommand(port: number): string { + const js = `require('http').createServer((q,r)=>r.end('ok')).listen(${port},'127.0.0.1')`; + return `${JSON.stringify(process.execPath)} -e ${JSON.stringify(js)}`; +} + +describe('withManagedServer', () => { + it('refuses to start when something already answers on the url', async () => { + const stranger = http.createServer((_, res) => res.end('not the sdk')); + const port = await listen(stranger); + let ran = false; + try { + await expect( + withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 5000, + async () => { + ran = true; + } + ) + ).rejects.toThrow(/already in use .*something answered/); + expect(ran).toBe(false); + } finally { + await new Promise((r) => stranger.close(r)); + } + }); + + it('refuses when a leftover accepts the connection but never answers', async () => { + const sockets = new Set(); + const staller = net.createServer((s) => sockets.add(s)); + const port = await new Promise((resolve) => + staller.listen(0, '127.0.0.1', () => + resolve((staller.address() as AddressInfo).port) + ) + ); + let ran = false; + try { + await expect( + withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 5000, + async () => { + ran = true; + } + ) + ).rejects.toThrow(/already in use .*did not answer within/); + expect(ran).toBe(false); + } finally { + for (const s of sockets) s.destroy(); + await new Promise((r) => staller.close(r)); + } + }); + + it('runs against the server it started when the port is free', async () => { + const port = await freePort(); + const result = await withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 10000, + async () => 'graded' + ); + expect(result).toBe('graded'); + }, 20000); +});