Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/sdk-runner/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,45 @@ function selfInvoke(args: string[], cwd: string): Promise<number> {
});
}

// Connect errors that mean nothing is listening at the address.
const NOBODY_LISTENING = new Set([
'ECONNREFUSED',
'EADDRNOTAVAIL',
'ENETUNREACH',
'EHOSTUNREACH'
]);

function nobodyListening(err: unknown): boolean {
const cause = (err as { cause?: { code?: string; errors?: unknown[] } })
?.cause;
if (!cause) return false;
// A localhost url tries ::1 and 127.0.0.1, and fetch reports both errors.
if (Array.isArray(cause.errors) && cause.errors.length > 0) {
return cause.errors.every((e) =>
NOBODY_LISTENING.has((e as { code?: string })?.code ?? '')
);
}
return NOBODY_LISTENING.has(cause.code ?? '');
}

// Returns why the url looks held, or null when nothing is listening there.
// Only a refused connect counts as free. A process that accepts the socket
// and then stalls or hangs up still holds the port.
async function heldBy(url: string, timeoutMs: number): Promise<string | null> {
try {
await fetch(url, { method: 'GET', signal: AbortSignal.timeout(timeoutMs) });
return 'something answered';
} catch (err) {
if (nobodyListening(err)) return null;
if ((err as Error)?.name === 'TimeoutError') {
return `something accepted the connection but did not answer within ${timeoutMs}ms`;
}
const cause = (err as { cause?: { code?: string; message?: string } })
?.cause;
return `the probe failed with ${cause?.code ?? cause?.message ?? String(err)}`;
}
}

async function waitForReady(url: string, timeoutMs: number): Promise<void> {
const deadline = Date.now() + timeoutMs;
// Per-probe timeout: a server that accepts the socket but never responds must
Expand Down Expand Up @@ -66,6 +105,16 @@ export async function withManagedServer<T>(
readyTimeoutMs: number,
fn: () => Promise<T>
): Promise<T> {
// A leftover process on the same port would pass the readiness probe, and
// the run would grade it instead of the server we are about to start.
const held = await heldBy(url, 1000);
if (held) {
throw new Error(
`${url} is already in use before the server started (${held}). ` +
`Stop that process or change the port, or this run would grade it ` +
`instead of the SDK's server.`
);
}
console.error(`[sdk] Starting server: ${command}`);
const child: ChildProcess = spawn(command, {
shell: true,
Expand Down
89 changes: 89 additions & 0 deletions src/sdk-runner/managed-server.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { describe, expect, it } from 'vitest';
import http from 'http';
import net from 'net';
import type { AddressInfo } from 'net';
import { withManagedServer } from './index';

function listen(server: http.Server, port = 0): Promise<number> {
return new Promise((resolve) => {
server.listen(port, '127.0.0.1', () =>
resolve((server.address() as AddressInfo).port)
);
});
}

async function freePort(): Promise<number> {
const s = http.createServer();
const port = await listen(s);
await new Promise((r) => s.close(r));
return port;
}

function serverCommand(port: number): string {
const js = `require('http').createServer((q,r)=>r.end('ok')).listen(${port},'127.0.0.1')`;
return `${JSON.stringify(process.execPath)} -e ${JSON.stringify(js)}`;
}

describe('withManagedServer', () => {
it('refuses to start when something already answers on the url', async () => {
const stranger = http.createServer((_, res) => res.end('not the sdk'));
const port = await listen(stranger);
let ran = false;
try {
await expect(
withManagedServer(
serverCommand(port),
process.cwd(),
`http://127.0.0.1:${port}/mcp`,
5000,
async () => {
ran = true;
}
)
).rejects.toThrow(/already in use .*something answered/);
expect(ran).toBe(false);
} finally {
await new Promise((r) => stranger.close(r));
}
});

it('refuses when a leftover accepts the connection but never answers', async () => {
const sockets = new Set<net.Socket>();
const staller = net.createServer((s) => sockets.add(s));
const port = await new Promise<number>((resolve) =>
staller.listen(0, '127.0.0.1', () =>
resolve((staller.address() as AddressInfo).port)
)
);
let ran = false;
try {
await expect(
withManagedServer(
serverCommand(port),
process.cwd(),
`http://127.0.0.1:${port}/mcp`,
5000,
async () => {
ran = true;
}
)
).rejects.toThrow(/already in use .*did not answer within/);
expect(ran).toBe(false);
} finally {
for (const s of sockets) s.destroy();
await new Promise((r) => staller.close(r));
}
});

it('runs against the server it started when the port is free', async () => {
const port = await freePort();
const result = await withManagedServer(
serverCommand(port),
process.cwd(),
`http://127.0.0.1:${port}/mcp`,
10000,
async () => 'graded'
);
expect(result).toBe('graded');
}, 20000);
});
Loading