From 32121fd1b8a3b15d21b146fded3e47c6731f5969 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 13 Nov 2026 12:23:13 -0800 Subject: [PATCH 1/2] fix(sdk-runner): refuse to start a managed server on a port that already answers withManagedServer spawned the SDK's server and then polled its url until something answered. A leftover process on that port answered first, so the run graded the leftover. The new server usually died on bind and ended the run partway through with "Server exited with code 1". A command that does not exit on a failed bind graded the stranger to the end. Probe the url once before spawning, and refuse with a message that names the url when anything answers. --- src/sdk-runner/index.ts | 19 +++++++++ src/sdk-runner/managed-server.test.ts | 60 +++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 src/sdk-runner/managed-server.test.ts diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts index bcd5b20a..8b516097 100644 --- a/src/sdk-runner/index.ts +++ b/src/sdk-runner/index.ts @@ -36,6 +36,16 @@ function selfInvoke(args: string[], cwd: string): Promise { }); } +// True when anything at all answers an HTTP GET on url within timeoutMs. +async function answers(url: string, timeoutMs: number): Promise { + try { + await fetch(url, { method: 'GET', signal: AbortSignal.timeout(timeoutMs) }); + return true; + } catch { + return false; + } +} + async function waitForReady(url: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; // Per-probe timeout: a server that accepts the socket but never responds must @@ -66,6 +76,15 @@ export async function withManagedServer( readyTimeoutMs: number, fn: () => Promise ): Promise { + // A leftover process on the same port would pass the readiness probe, and + // the run would grade it instead of the server we are about to start. + if (await answers(url, 1000)) { + throw new Error( + `Something is already answering at ${url} before the server started. ` + + `Stop that process or change the port, or this run would grade it ` + + `instead of the SDK's server.` + ); + } console.error(`[sdk] Starting server: ${command}`); const child: ChildProcess = spawn(command, { shell: true, diff --git a/src/sdk-runner/managed-server.test.ts b/src/sdk-runner/managed-server.test.ts new file mode 100644 index 00000000..a88d1022 --- /dev/null +++ b/src/sdk-runner/managed-server.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import http from 'http'; +import type { AddressInfo } from 'net'; +import { withManagedServer } from './index'; + +function listen(server: http.Server, port = 0): Promise { + return new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => + resolve((server.address() as AddressInfo).port) + ); + }); +} + +async function freePort(): Promise { + const s = http.createServer(); + const port = await listen(s); + await new Promise((r) => s.close(r)); + return port; +} + +function serverCommand(port: number): string { + const js = `require('http').createServer((q,r)=>r.end('ok')).listen(${port},'127.0.0.1')`; + return `${JSON.stringify(process.execPath)} -e ${JSON.stringify(js)}`; +} + +describe('withManagedServer', () => { + it('refuses to start when something already answers on the url', async () => { + const stranger = http.createServer((_, res) => res.end('not the sdk')); + const port = await listen(stranger); + let ran = false; + try { + await expect( + withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 5000, + async () => { + ran = true; + } + ) + ).rejects.toThrow(/already answering/); + expect(ran).toBe(false); + } finally { + await new Promise((r) => stranger.close(r)); + } + }); + + it('runs against the server it started when the port is free', async () => { + const port = await freePort(); + const result = await withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 10000, + async () => 'graded' + ); + expect(result).toBe('graded'); + }, 20000); +}); From 88c9ea846c839ee4669b62a637eaa95a9bb12473 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 13 Nov 2026 14:50:10 -0800 Subject: [PATCH 2/2] fix(sdk-runner): count only a refused connect as a free port A leftover that accepts the connection and then stalls made the probe time out. The old check read a timeout as a free port, so the run went ahead and later failed with "Server exited with code 1". The probe now counts the port as free only on ECONNREFUSED and the other nobody-listening codes. A localhost url reports one error for ::1 and one for 127.0.0.1, and both must say nobody is listening. The error message now says why the port looks held. --- src/sdk-runner/index.ts | 44 ++++++++++++++++++++++----- src/sdk-runner/managed-server.test.ts | 31 ++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/src/sdk-runner/index.ts b/src/sdk-runner/index.ts index 8b516097..53a3221c 100644 --- a/src/sdk-runner/index.ts +++ b/src/sdk-runner/index.ts @@ -36,13 +36,42 @@ function selfInvoke(args: string[], cwd: string): Promise { }); } -// True when anything at all answers an HTTP GET on url within timeoutMs. -async function answers(url: string, timeoutMs: number): Promise { +// Connect errors that mean nothing is listening at the address. +const NOBODY_LISTENING = new Set([ + 'ECONNREFUSED', + 'EADDRNOTAVAIL', + 'ENETUNREACH', + 'EHOSTUNREACH' +]); + +function nobodyListening(err: unknown): boolean { + const cause = (err as { cause?: { code?: string; errors?: unknown[] } }) + ?.cause; + if (!cause) return false; + // A localhost url tries ::1 and 127.0.0.1, and fetch reports both errors. + if (Array.isArray(cause.errors) && cause.errors.length > 0) { + return cause.errors.every((e) => + NOBODY_LISTENING.has((e as { code?: string })?.code ?? '') + ); + } + return NOBODY_LISTENING.has(cause.code ?? ''); +} + +// Returns why the url looks held, or null when nothing is listening there. +// Only a refused connect counts as free. A process that accepts the socket +// and then stalls or hangs up still holds the port. +async function heldBy(url: string, timeoutMs: number): Promise { try { await fetch(url, { method: 'GET', signal: AbortSignal.timeout(timeoutMs) }); - return true; - } catch { - return false; + return 'something answered'; + } catch (err) { + if (nobodyListening(err)) return null; + if ((err as Error)?.name === 'TimeoutError') { + return `something accepted the connection but did not answer within ${timeoutMs}ms`; + } + const cause = (err as { cause?: { code?: string; message?: string } }) + ?.cause; + return `the probe failed with ${cause?.code ?? cause?.message ?? String(err)}`; } } @@ -78,9 +107,10 @@ export async function withManagedServer( ): Promise { // A leftover process on the same port would pass the readiness probe, and // the run would grade it instead of the server we are about to start. - if (await answers(url, 1000)) { + const held = await heldBy(url, 1000); + if (held) { throw new Error( - `Something is already answering at ${url} before the server started. ` + + `${url} is already in use before the server started (${held}). ` + `Stop that process or change the port, or this run would grade it ` + `instead of the SDK's server.` ); diff --git a/src/sdk-runner/managed-server.test.ts b/src/sdk-runner/managed-server.test.ts index a88d1022..37f64ef5 100644 --- a/src/sdk-runner/managed-server.test.ts +++ b/src/sdk-runner/managed-server.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import http from 'http'; +import net from 'net'; import type { AddressInfo } from 'net'; import { withManagedServer } from './index'; @@ -39,13 +40,41 @@ describe('withManagedServer', () => { ran = true; } ) - ).rejects.toThrow(/already answering/); + ).rejects.toThrow(/already in use .*something answered/); expect(ran).toBe(false); } finally { await new Promise((r) => stranger.close(r)); } }); + it('refuses when a leftover accepts the connection but never answers', async () => { + const sockets = new Set(); + const staller = net.createServer((s) => sockets.add(s)); + const port = await new Promise((resolve) => + staller.listen(0, '127.0.0.1', () => + resolve((staller.address() as AddressInfo).port) + ) + ); + let ran = false; + try { + await expect( + withManagedServer( + serverCommand(port), + process.cwd(), + `http://127.0.0.1:${port}/mcp`, + 5000, + async () => { + ran = true; + } + ) + ).rejects.toThrow(/already in use .*did not answer within/); + expect(ran).toBe(false); + } finally { + for (const s of sockets) s.destroy(); + await new Promise((r) => staller.close(r)); + } + }); + it('runs against the server it started when the port is free', async () => { const port = await freePort(); const result = await withManagedServer(