diff --git a/README.md b/README.md index d5c4cb3f..ff82d8e0 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ npx @modelcontextprotocol/conformance server --url [--scenario ] - `--scenario ` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default - `--suite ` - Suite to run: "active" (default; excludes pending and draft-spec scenarios), "all", "draft" (scenarios targeting the in-progress draft spec), or "pending" - `--requirements ` - Run exactly what a spec revision requires, frozen at its release (see [Conformance Requirements](#conformance-requirements)) +- `--production` - Test a capability-partial production server rather than an SDK reference fixture: scenarios requiring optional capabilities or extensions the server did not declare, or reference-fixture artifacts (`test_*` / `test://`), are reported as `SKIPPED` instead of `FAILURE` - `--expected-failures ` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures)) - `--verbose` - Show verbose output diff --git a/src/connection/index.ts b/src/connection/index.ts index 47952c64..cfc57909 100644 --- a/src/connection/index.ts +++ b/src/connection/index.ts @@ -83,6 +83,11 @@ export interface Connection { export interface RunContext { serverUrl: string; specVersion: SpecVersion; + /** + * True when running in opt-in `--production` mode against a real deployment + * rather than an SDK reference fixture server. + */ + production?: boolean; /** * Open a version-appropriate connection to the server-under-test. * Scenarios that test the connection mechanics themselves (initialize, diff --git a/src/index.ts b/src/index.ts index b376644e..fbab02f7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -555,6 +555,10 @@ program '--requirements ', 'Run exactly the scenarios a spec revision requires, frozen at its release (e.g. 2026-07-28). Replaces --suite and --spec-version' ) + .option( + '--production', + 'Test a production server: skip scenarios requiring undeclared optional capabilities or reference fixtures' + ) .option('--timeout ', 'Per-scenario timeout in milliseconds', '30000') .option('--verbose', 'Show verbose output (JSON instead of pretty print)') .action(async (options, cmd) => { @@ -564,6 +568,7 @@ program const timeout = parseInt(options.timeout, 10); const verbose = options.verbose ?? false; + const production = options.production ?? false; const outputDir = options.outputDir; const requirements = resolveRequirements(options.requirements, { specVersion: options.specVersion, @@ -591,7 +596,8 @@ program outputDir, specVersionFilter, options.force ?? false, - timeout + timeout, + { production } ); // Inapplicable scenario/spec-version combination (already logged by @@ -664,6 +670,9 @@ program const allResults: { scenario: string; checks: ConformanceCheck[] }[] = []; + const capabilitiesCache = production + ? new Map>() + : undefined; for (const scenarioName of scenarios) { console.log(`\n=== Running scenario: ${scenarioName} ===`); @@ -679,7 +688,8 @@ program // a requirement set decides membership, so its choice outranks a // scenario's own applicability window at the pinned revision Boolean(requirements), - timeout + timeout, + { production, capabilitiesCache } ) ); allResults.push({ scenario: scenarioName, checks: result.checks }); diff --git a/src/runner/index.ts b/src/runner/index.ts index a39a86ca..275810b2 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -10,7 +10,8 @@ export { export { runServerConformanceTest, printServerResults, - printServerSummary + printServerSummary, + type ServerRunOptions } from './server'; // Export utilities diff --git a/src/runner/server.ts b/src/runner/server.ts index 21e80ec4..011e2997 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -2,18 +2,37 @@ import { promises as fs } from 'fs'; import path from 'path'; import { ConformanceCheck, + ScenarioSource, SpecVersion, LATEST_SPEC_VERSION, DRAFT_PROTOCOL_VERSION } from '../types'; import { getClientScenario, isScenarioApplicableAt } from '../scenarios'; import { connectFor, type RunContext } from '../connection'; +import { + evaluateScenarioRequirements, + isPlainObject +} from '../scenarios/server/requirements-check'; import { resetWireValidation, wireSchemaChecks } from '../validation/wire-schema'; import { createResultDir, formatPrettyChecks } from './utils'; +export interface ServerRunOptions { + /** + * When true, evaluate each scenario's declarative `requires` metadata + * against the server's advertised capabilities and emit `SKIPPED` checks + * for undeclared optional capabilities/extensions or fixture contracts. + */ + production?: boolean; + /** + * Optional per-spec-version capability cache shared across scenarios in a + * suite run so capability discovery happens at most once per wire version. + */ + capabilitiesCache?: Map>; +} + /** * Format markdown-style text for terminal output using ANSI codes */ @@ -73,13 +92,142 @@ async function runScenarioBounded( ]; } +async function discoverServerCapabilities( + serverUrl: string, + specVersion: SpecVersion, + timeout: number, + cache?: Map> +): Promise | null> { + if (cache?.has(specVersion)) { + return cache.get(specVersion)!; + } + + let timeoutHandle: NodeJS.Timeout | undefined; + const probe = (async () => { + const conn = await connectFor(specVersion)(serverUrl); + try { + const discovered = await conn.discover(); + return isPlainObject(discovered.capabilities) + ? discovered.capabilities + : {}; + } finally { + await conn.close().catch(() => {}); + } + })(); + probe.catch(() => {}); + + try { + const result = await Promise.race([ + probe, + new Promise((resolve) => { + timeoutHandle = setTimeout(() => resolve(null), timeout); + }) + ]); + if (result !== null) { + cache?.set(specVersion, result); + } + return result; + } catch { + // Discovery failed (unreachable server or broken handshake). Return null + // so the scenario executes and surfaces the real protocol/network failure + // rather than falsely skipping. + return null; + } finally { + clearTimeout(timeoutHandle); + } +} + +async function writeChecksIfNeeded( + resultDir: string | undefined, + checks: ConformanceCheck[] +): Promise { + if (!resultDir) return; + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); + console.log(`Results saved to ${resultDir}`); +} + +function logSpecVersionSkipIfNeeded( + scenarioName: string, + source: ScenarioSource, + specVersion: SpecVersion | undefined, + force: boolean +): boolean { + if ( + specVersion === undefined || + force || + isScenarioApplicableAt(source, specVersion) + ) { + return false; + } + + const introduced = + 'introducedIn' in source + ? `introduced in ${source.introducedIn}` + + (source.removedIn !== undefined + ? `, removed in ${source.removedIn}` + : '') + : 'extension scenario, not on the spec timeline'; + console.log( + `SKIPPED: scenario '${scenarioName}' is not applicable at spec version ` + + `${specVersion} (${introduced}). Use --force to run it anyway.` + ); + return true; +} + +function resolveDefaultSpecVersion( + source: ScenarioSource, + specVersion?: SpecVersion +): SpecVersion { + if (specVersion) return specVersion; + return 'extensionId' in source || + source.introducedIn === DRAFT_PROTOCOL_VERSION + ? DRAFT_PROTOCOL_VERSION + : LATEST_SPEC_VERSION; +} + +async function evaluateProductionSkip( + serverUrl: string, + scenarioName: string, + requires: NonNullable>['requires'], + resolvedSpecVersion: SpecVersion, + timeout: number, + options?: ServerRunOptions +): Promise { + if (!options?.production || !requires) return null; + resetWireValidation(); + const capabilities = await discoverServerCapabilities( + serverUrl, + resolvedSpecVersion, + timeout, + options.capabilitiesCache + ); + if (capabilities === null) return null; + + const skippedCheck = evaluateScenarioRequirements( + scenarioName, + requires, + capabilities + ); + if (!skippedCheck) return null; + + const discoverySchemaFailures = wireSchemaChecks(resolvedSpecVersion).filter( + (c) => c.status === 'FAILURE' + ); + console.log(`SKIPPED: ${skippedCheck.description}`); + return [skippedCheck, ...discoverySchemaFailures]; +} + export async function runServerConformanceTest( serverUrl: string, scenarioName: string, outputDir?: string, specVersion?: SpecVersion, force = false, - timeout: number = 30000 + timeout: number = 30000, + options?: ServerRunOptions ): Promise<{ checks: ConformanceCheck[]; resultDir?: string; @@ -87,34 +235,20 @@ export async function runServerConformanceTest( skipped?: boolean; }> { let resultDir: string | undefined; - if (outputDir) { resultDir = createResultDir(outputDir, scenarioName, 'server'); await fs.mkdir(resultDir, { recursive: true }); } - // Scenario is guaranteed to exist by CLI validation const scenario = getClientScenario(scenarioName)!; - - // An explicitly-requested spec version outside the scenario's applicability - // window is a contradiction: running anyway would test something other than - // what the flag claims. Skip (exit 0) unless --force. if ( - specVersion !== undefined && - !force && - !isScenarioApplicableAt(scenario.source, specVersion) + logSpecVersionSkipIfNeeded( + scenarioName, + scenario.source, + specVersion, + force + ) ) { - const introduced = - 'introducedIn' in scenario.source - ? `introduced in ${scenario.source.introducedIn}` + - (scenario.source.removedIn !== undefined - ? `, removed in ${scenario.source.removedIn}` - : '') - : 'extension scenario, not on the spec timeline'; - console.log( - `SKIPPED: scenario '${scenarioName}' is not applicable at spec version ` + - `${specVersion} (${introduced}). Use --force to run it anyway.` - ); return { checks: [], resultDir, @@ -123,25 +257,34 @@ export async function runServerConformanceTest( }; } - // When --spec-version is omitted, infer the version from the scenario's - // declared source so draft-only scenarios get the draft (stateless) - // connection rather than the stateful latest-spec default. Extension - // scenarios are off-timeline; today every extension in this repo lives - // on draft, so they fall under the same inference. - const resolvedSpecVersion = - specVersion ?? - ('extensionId' in scenario.source || - scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION - ? DRAFT_PROTOCOL_VERSION - : LATEST_SPEC_VERSION); + const resolvedSpecVersion = resolveDefaultSpecVersion( + scenario.source, + specVersion + ); + const skippedChecks = await evaluateProductionSkip( + serverUrl, + scenarioName, + scenario.requires, + resolvedSpecVersion, + timeout, + options + ); + if (skippedChecks) { + await writeChecksIfNeeded(resultDir, skippedChecks); + return { + checks: skippedChecks, + resultDir, + scenarioDescription: scenario.description + }; + } console.log( `Running client scenario '${scenarioName}' against server: ${serverUrl}` ); - const ctx: RunContext = { serverUrl, specVersion: resolvedSpecVersion, + production: Boolean(options?.production), connect: (opts) => connectFor(resolvedSpecVersion)(serverUrl, opts) }; resetWireValidation(); @@ -151,21 +294,9 @@ export async function runServerConformanceTest( timeout ); checks.push(...wireSchemaChecks(resolvedSpecVersion)); + await writeChecksIfNeeded(resultDir, checks); - if (resultDir) { - await fs.writeFile( - path.join(resultDir, 'checks.json'), - JSON.stringify(checks, null, 2) - ); - - console.log(`Results saved to ${resultDir}`); - } - - return { - checks, - resultDir, - scenarioDescription: scenario.description - }; + return { checks, resultDir, scenarioDescription: scenario.description }; } export function printServerResults( @@ -175,6 +306,7 @@ export function printServerResults( ): { passed: number; failed: number; + skipped: number; denominator: number; warnings: number; } { @@ -183,6 +315,7 @@ export function printServerResults( ).length; const passed = checks.filter((c) => c.status === 'SUCCESS').length; const failed = checks.filter((c) => c.status === 'FAILURE').length; + const skipped = checks.filter((c) => c.status === 'SKIPPED').length; const warnings = checks.filter((c) => c.status === 'WARNING').length; if (verbose) { @@ -191,9 +324,10 @@ export function printServerResults( console.log(`Checks:\n${formatPrettyChecks(checks)}`); } + const skippedSuffix = skipped > 0 ? `, ${skipped} skipped` : ''; console.log(`\nTest Results:`); console.log( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` + `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings${skippedSuffix}` ); if (failed > 0) { @@ -209,29 +343,42 @@ export function printServerResults( }); } - return { passed, failed, denominator, warnings }; + return { passed, failed, skipped, denominator, warnings }; } export function printServerSummary( allResults: { scenario: string; checks: ConformanceCheck[] }[] -): { totalPassed: number; totalFailed: number } { +): { totalPassed: number; totalFailed: number; totalSkipped: number } { console.log('\n\n=== SUMMARY ==='); let totalPassed = 0; let totalFailed = 0; + let totalSkipped = 0; for (const result of allResults) { const passed = result.checks.filter((c) => c.status === 'SUCCESS').length; const failed = result.checks.filter((c) => c.status === 'FAILURE').length; + const skipped = result.checks.filter((c) => c.status === 'SKIPPED').length; totalPassed += passed; totalFailed += failed; + totalSkipped += skipped; + + if (passed === 0 && failed === 0 && skipped > 0) { + console.log(`- ${result.scenario}: ${skipped} skipped`); + continue; + } const status = failed === 0 ? '✓' : '✗'; + const skippedSuffix = skipped > 0 ? `, ${skipped} skipped` : ''; console.log( - `${status} ${result.scenario}: ${passed} passed, ${failed} failed` + `${status} ${result.scenario}: ${passed} passed, ${failed} failed${skippedSuffix}` ); } - console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`); + const totalSkippedSuffix = + totalSkipped > 0 ? `, ${totalSkipped} skipped` : ''; + console.log( + `\nTotal: ${totalPassed} passed, ${totalFailed} failed${totalSkippedSuffix}` + ); - return { totalPassed, totalFailed }; + return { totalPassed, totalFailed, totalSkipped }; } diff --git a/src/scenarios/server/caching.ts b/src/scenarios/server/caching.ts index c7f4a5b5..3e35e519 100644 --- a/src/scenarios/server/caching.ts +++ b/src/scenarios/server/caching.ts @@ -16,6 +16,8 @@ import { type RunContext } from '../../connection'; import type { CacheableResult } from '../../spec-types/draft'; +import { notTestable } from '../untestable'; +import { hasServerCapability, isPlainObject } from './requirements-check'; const SPEC_REFS = [ { @@ -28,6 +30,8 @@ const SPEC_REFS = [ } ]; +type CacheableCapability = 'tools' | 'prompts' | 'resources'; + interface CachingFields { ttlMs: unknown; cacheScope: unknown; @@ -35,14 +39,17 @@ interface CachingFields { hasCacheScope: boolean; } +interface EndpointFields { + endpoint: string; + fields: CachingFields; +} + function extractCachingFields(result: Partial): CachingFields { - const hasTtlMs = 'ttlMs' in result; - const hasCacheScope = 'cacheScope' in result; return { ttlMs: result.ttlMs, cacheScope: result.cacheScope, - hasTtlMs, - hasCacheScope + hasTtlMs: 'ttlMs' in result, + hasCacheScope: 'cacheScope' in result }; } @@ -53,7 +60,6 @@ function buildPresenceCheck( fields: CachingFields ): ConformanceCheck { const errors: string[] = []; - if (!fields.hasTtlMs) { errors.push(`${endpoint} response missing ttlMs`); } @@ -78,164 +84,257 @@ function buildPresenceCheck( }; } -export class CachingScenario implements ClientScenario { - name = 'caching'; - readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; - description = `Test that servers include caching hints (ttlMs and cacheScope) on cacheable results (SEP-2549). +function buildSkippedCapabilityCheck( + id: string, + name: string, + endpoint: string, + capability: CacheableCapability +): ConformanceCheck { + return { + id, + name, + description: `${endpoint} response includes ttlMs and cacheScope caching hints`, + status: 'SKIPPED', + timestamp: new Date().toISOString(), + errorMessage: `${endpoint} was not exercised: server did not declare '${capability}' capability.`, + specReferences: SPEC_REFS, + details: { + skippedReason: 'missing-capability', + capability + } + }; +} -**Server Implementation Requirements:** +async function discoverDeclaredCapabilities( + conn: Connection +): Promise | null> { + try { + const discovered = await conn.discover(); + return isPlainObject(discovered.capabilities) + ? discovered.capabilities + : {}; + } catch { + // When discovery itself fails, do not assume capabilities are absent: + // probe all endpoints so a broken server fails rather than skipping. + return null; + } +} -Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "private") on results from: -- \`tools/list\` -- \`prompts/list\` -- \`resources/list\` -- \`resources/templates/list\` -- \`resources/read\``; +function isCapabilityDeclared( + capabilities: Record | null, + capability: CacheableCapability +): boolean { + return capabilities === null || hasServerCapability(capabilities, capability); +} - async run(ctx: RunContext): Promise { - const checks: ConformanceCheck[] = []; - const allFields: Array<{ endpoint: string; fields: CachingFields }> = []; +async function queryCacheableEndpoint( + conn: Connection, + checks: ConformanceCheck[], + allFields: EndpointFields[], + checkId: string, + checkName: string, + endpoint: string, + params?: Record +): Promise<(CacheableResult & Record) | undefined> { + const description = `${endpoint} response includes ttlMs and cacheScope caching hints`; + try { + const result = await conn.request< + CacheableResult & Record + >(endpoint, params); + const fields = extractCachingFields(result); + allFields.push({ endpoint, fields }); + checks.push(buildPresenceCheck(checkId, checkName, endpoint, fields)); + return result; + } catch (error) { + const rpcError = error instanceof JsonRpcError ? error : undefined; + checks.push({ + id: checkId, + name: checkName, + description, + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: rpcError + ? `${endpoint} returned JSON-RPC error ${rpcError.code}: ${rpcError.message}` + : `${endpoint} request failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: SPEC_REFS, + details: rpcError + ? { error: { code: rpcError.code, message: rpcError.message } } + : undefined + }); + return undefined; + } +} - // SEP-2549 only exists in the draft spec, so each cacheable endpoint is - // queried over the version-appropriate connection. Under --spec-version - // draft that resolves to the stateless impl (SEP-2575): protocolVersion - // 2026-07-28 plus the cross-cutting _meta and standard headers - // (issue #315). - let conn: Connection; - try { - conn = await ctx.connect(); - } catch (error) { - checks.push({ - id: 'sep-2549-caching-connection', - name: 'CachingConnection', - description: 'Caching hints scenario failed to connect', - status: 'FAILURE', - timestamp: new Date().toISOString(), - errorMessage: `Connection failed: ${error instanceof Error ? error.message : String(error)}`, - specReferences: SPEC_REFS - }); - return checks; - } +async function probeResourceEndpoints( + conn: Connection, + checks: ConformanceCheck[], + allFields: EndpointFields[], + resourcesDeclared: boolean +): Promise { + if (!resourcesDeclared) { + checks.push( + buildSkippedCapabilityCheck( + 'sep-2549-resources-list-caching-hints', + 'ResourcesListCachingHints', + 'resources/list', + 'resources' + ), + buildSkippedCapabilityCheck( + 'sep-2549-resources-templates-list-caching-hints', + 'ResourcesTemplatesListCachingHints', + 'resources/templates/list', + 'resources' + ), + buildSkippedCapabilityCheck( + 'sep-2549-resources-read-caching-hints', + 'ResourcesReadCachingHints', + 'resources/read', + 'resources' + ) + ); + return; + } - const queryEndpoint = async ( - checkId: string, - checkName: string, - endpoint: string, - params?: Record - ): Promise<(CacheableResult & Record) | undefined> => { - const description = `${endpoint} response includes ttlMs and cacheScope caching hints`; - try { - const result = await conn.request< - CacheableResult & Record - >(endpoint, params); - const fields = extractCachingFields(result); - allFields.push({ endpoint, fields }); - checks.push(buildPresenceCheck(checkId, checkName, endpoint, fields)); - return result; - } catch (error) { - const rpcError = error instanceof JsonRpcError ? error : undefined; - checks.push({ - id: checkId, - name: checkName, - description, - status: 'FAILURE', - timestamp: new Date().toISOString(), - errorMessage: rpcError - ? `${endpoint} returned JSON-RPC error ${rpcError.code}: ${rpcError.message}` - : `${endpoint} request failed: ${error instanceof Error ? error.message : String(error)}`, - specReferences: SPEC_REFS, - details: rpcError - ? { error: { code: rpcError.code, message: rpcError.message } } - : undefined - }); - return undefined; - } - }; + const resourcesResult = await queryCacheableEndpoint( + conn, + checks, + allFields, + 'sep-2549-resources-list-caching-hints', + 'ResourcesListCachingHints', + 'resources/list' + ); + await queryCacheableEndpoint( + conn, + checks, + allFields, + 'sep-2549-resources-templates-list-caching-hints', + 'ResourcesTemplatesListCachingHints', + 'resources/templates/list' + ); - // 1. tools/list - await queryEndpoint( - 'sep-2549-tools-list-caching-hints', - 'ToolsListCachingHints', - 'tools/list' + const resources = resourcesResult?.resources as + | Array<{ uri?: string }> + | undefined; + const firstResourceUri = resources?.[0]?.uri; + if (firstResourceUri) { + await queryCacheableEndpoint( + conn, + checks, + allFields, + 'sep-2549-resources-read-caching-hints', + 'ResourcesReadCachingHints', + 'resources/read', + { uri: firstResourceUri } ); + return; + } - // 2. prompts/list - await queryEndpoint( - 'sep-2549-prompts-list-caching-hints', - 'PromptsListCachingHints', - 'prompts/list' - ); + // Keep the emitted check-ID set stable even when there is nothing to read. + checks.push({ + id: 'sep-2549-resources-read-caching-hints', + name: 'ResourcesReadCachingHints', + description: + 'resources/read response includes ttlMs and cacheScope caching hints', + status: 'SKIPPED', + timestamp: new Date().toISOString(), + errorMessage: + 'resources/read was not exercised: resources/list failed or returned no resources.', + specReferences: SPEC_REFS + }); +} - // 3. resources/list - const resourcesResult = await queryEndpoint( - 'sep-2549-resources-list-caching-hints', - 'ResourcesListCachingHints', - 'resources/list' - ); +function validateTtlValues(allFields: EndpointFields[]): string[] { + const endpointsWithTtl = allFields.filter((f) => f.fields.hasTtlMs); + if (endpointsWithTtl.length === 0) { + return ['no endpoints returned ttlMs']; + } - // 4. resources/templates/list - await queryEndpoint( - 'sep-2549-resources-templates-list-caching-hints', - 'ResourcesTemplatesListCachingHints', - 'resources/templates/list' - ); + const errors: string[] = []; + for (const { endpoint, fields } of endpointsWithTtl) { + const val = fields.ttlMs; + if (typeof val !== 'number') { + errors.push(`${endpoint}: ttlMs is ${typeof val}, expected number`); + } else if (!Number.isInteger(val)) { + errors.push(`${endpoint}: ttlMs is ${val}, expected integer`); + } else if (val < 0) { + errors.push(`${endpoint}: ttlMs is ${val}, must be >= 0`); + } + } + return errors; +} - // 5. resources/read — use first resource from resources/list - const resources = resourcesResult?.resources as - | Array<{ uri?: string }> - | undefined; - const firstResourceUri = resources?.[0]?.uri; - if (firstResourceUri) { - await queryEndpoint( - 'sep-2549-resources-read-caching-hints', - 'ResourcesReadCachingHints', - 'resources/read', - { uri: firstResourceUri } +function validateCacheScopeValues(allFields: EndpointFields[]): string[] { + const endpointsWithScope = allFields.filter((f) => f.fields.hasCacheScope); + if (endpointsWithScope.length === 0) { + return ['no endpoints returned cacheScope']; + } + + const errors: string[] = []; + for (const { endpoint, fields } of endpointsWithScope) { + const val = fields.cacheScope; + if (val !== 'public' && val !== 'private') { + errors.push( + `${endpoint}: cacheScope is ${JSON.stringify(val)}, expected "public" or "private"` ); - } else { - // Keep the emitted check-ID set stable even when there is nothing to - // read (resources/list failed or the server exposes no resources). - checks.push({ - id: 'sep-2549-resources-read-caching-hints', - name: 'ResourcesReadCachingHints', - description: - 'resources/read response includes ttlMs and cacheScope caching hints', - status: 'SKIPPED', - timestamp: new Date().toISOString(), - errorMessage: - 'resources/read was not exercised: resources/list failed or returned no resources.', - specReferences: SPEC_REFS - }); } + } + return errors; +} - await conn.close(); - - // 6. Aggregate: ttlMs must be a non-negative integer - const ttlErrors: string[] = []; - const endpointsWithTtl = allFields.filter((f) => f.fields.hasTtlMs); - if (endpointsWithTtl.length === 0) { - ttlErrors.push('no endpoints returned ttlMs'); - } else { - for (const { endpoint, fields } of endpointsWithTtl) { - const val = fields.ttlMs; - if (typeof val !== 'number') { - ttlErrors.push( - `${endpoint}: ttlMs is ${typeof val}, expected number` - ); - } else if (!Number.isInteger(val)) { - ttlErrors.push(`${endpoint}: ttlMs is ${val}, expected integer`); - } else if (val < 0) { - ttlErrors.push(`${endpoint}: ttlMs is ${val}, must be >= 0`); +function buildAggregateChecks( + allFields: EndpointFields[], + anyCapabilityDeclared: boolean, + production: boolean +): ConformanceCheck[] { + const timestamp = new Date().toISOString(); + if (!anyCapabilityDeclared) { + const status = production ? 'SKIPPED' : 'FAILURE'; + const message = production + ? 'Skipped in production mode: server did not declare tools, prompts, or resources capabilities.' + : notTestable( + 'server declared none of tools, prompts, or resources capabilities, so caching hints could not be exercised' + ); + const details = production + ? { + skippedReason: 'missing-capability', + requiredAnyOf: ['tools', 'prompts', 'resources'] } + : { untestable: true, endpoints: [] }; + + return [ + { + id: 'sep-2549-ttl-non-negative', + name: 'TtlNonNegative', + description: 'All ttlMs values are non-negative integers', + status, + timestamp, + errorMessage: message, + specReferences: SPEC_REFS, + details + }, + { + id: 'sep-2549-cache-scope-valid', + name: 'CacheScopeValid', + description: 'All cacheScope values are "public" or "private"', + status, + timestamp, + errorMessage: message, + specReferences: SPEC_REFS, + details } - } + ]; + } - checks.push({ + const ttlErrors = validateTtlValues(allFields); + const scopeErrors = validateCacheScopeValues(allFields); + + return [ + { id: 'sep-2549-ttl-non-negative', name: 'TtlNonNegative', description: 'All ttlMs values are non-negative integers', status: ttlErrors.length === 0 ? 'SUCCESS' : 'FAILURE', - timestamp: new Date().toISOString(), + timestamp, errorMessage: ttlErrors.length > 0 ? ttlErrors.join('; ') : undefined, specReferences: SPEC_REFS, details: { @@ -244,30 +343,13 @@ Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "p ttlMs: f.fields.ttlMs })) } - }); - - // 7. Aggregate: cacheScope must be "public" or "private" - const scopeErrors: string[] = []; - const endpointsWithScope = allFields.filter((f) => f.fields.hasCacheScope); - if (endpointsWithScope.length === 0) { - scopeErrors.push('no endpoints returned cacheScope'); - } else { - for (const { endpoint, fields } of endpointsWithScope) { - const val = fields.cacheScope; - if (val !== 'public' && val !== 'private') { - scopeErrors.push( - `${endpoint}: cacheScope is ${JSON.stringify(val)}, expected "public" or "private"` - ); - } - } - } - - checks.push({ + }, + { id: 'sep-2549-cache-scope-valid', name: 'CacheScopeValid', description: 'All cacheScope values are "public" or "private"', status: scopeErrors.length === 0 ? 'SUCCESS' : 'FAILURE', - timestamp: new Date().toISOString(), + timestamp, errorMessage: scopeErrors.length > 0 ? scopeErrors.join('; ') : undefined, specReferences: SPEC_REFS, details: { @@ -276,8 +358,102 @@ Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "p cacheScope: f.fields.cacheScope })) } - }); + } + ]; +} +export class CachingScenario implements ClientScenario { + name = 'caching'; + readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { + anyCapability: ['tools', 'prompts', 'resources'] + } as const; + description = `Test that servers include caching hints (ttlMs and cacheScope) on cacheable results (SEP-2549). + +**Server Implementation Requirements:** + +Servers MUST include \`ttlMs\` (integer >= 0) and \`cacheScope\` ("public" or "private") on results from: +- \`tools/list\` +- \`prompts/list\` +- \`resources/list\` +- \`resources/templates/list\` +- \`resources/read\``; + + async run(ctx: RunContext): Promise { + const checks: ConformanceCheck[] = []; + const allFields: EndpointFields[] = []; + + let conn: Connection; + try { + conn = await ctx.connect(); + } catch (error) { + checks.push({ + id: 'sep-2549-caching-connection', + name: 'CachingConnection', + description: 'Caching hints scenario failed to connect', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Connection failed: ${error instanceof Error ? error.message : String(error)}`, + specReferences: SPEC_REFS + }); + return checks; + } + + const capabilities = await discoverDeclaredCapabilities(conn); + const hasTools = isCapabilityDeclared(capabilities, 'tools'); + const hasPrompts = isCapabilityDeclared(capabilities, 'prompts'); + const hasResources = isCapabilityDeclared(capabilities, 'resources'); + + if (hasTools) { + await queryCacheableEndpoint( + conn, + checks, + allFields, + 'sep-2549-tools-list-caching-hints', + 'ToolsListCachingHints', + 'tools/list' + ); + } else { + checks.push( + buildSkippedCapabilityCheck( + 'sep-2549-tools-list-caching-hints', + 'ToolsListCachingHints', + 'tools/list', + 'tools' + ) + ); + } + + if (hasPrompts) { + await queryCacheableEndpoint( + conn, + checks, + allFields, + 'sep-2549-prompts-list-caching-hints', + 'PromptsListCachingHints', + 'prompts/list' + ); + } else { + checks.push( + buildSkippedCapabilityCheck( + 'sep-2549-prompts-list-caching-hints', + 'PromptsListCachingHints', + 'prompts/list', + 'prompts' + ) + ); + } + + await probeResourceEndpoints(conn, checks, allFields, hasResources); + await conn.close(); + + checks.push( + ...buildAggregateChecks( + allFields, + hasTools || hasPrompts || hasResources, + Boolean(ctx.production) + ) + ); return checks; } } diff --git a/src/scenarios/server/elicitation-defaults.ts b/src/scenarios/server/elicitation-defaults.ts index 20ed8ee9..9b915dc3 100644 --- a/src/scenarios/server/elicitation-defaults.ts +++ b/src/scenarios/server/elicitation-defaults.ts @@ -17,6 +17,7 @@ export class ElicitationDefaultsScenario implements ClientScenario { introducedIn: '2025-11-25', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test elicitation with default values for all primitive types (SEP-1034). **Server Implementation Requirements:** diff --git a/src/scenarios/server/elicitation-enums.ts b/src/scenarios/server/elicitation-enums.ts index ac048600..0390b44a 100644 --- a/src/scenarios/server/elicitation-enums.ts +++ b/src/scenarios/server/elicitation-enums.ts @@ -17,6 +17,7 @@ export class ElicitationEnumsScenario implements ClientScenario { introducedIn: '2025-11-25', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test elicitation with enum schema improvements (SEP-1330). **Server Implementation Requirements:** diff --git a/src/scenarios/server/http-standard-headers.ts b/src/scenarios/server/http-standard-headers.ts index d5ca0320..e10f601b 100644 --- a/src/scenarios/server/http-standard-headers.ts +++ b/src/scenarios/server/http-standard-headers.ts @@ -126,6 +126,23 @@ function untestableMcpNameCases(checks: ConformanceCheck[], reason: string) { } } +/** Emit the Mcp-Name cases as SKIPPED in production mode to avoid invoking arbitrary domain tools. */ +function skippedMcpNameCases(checks: ConformanceCheck[], reason: string) { + const timestamp = new Date().toISOString(); + for (const c of MCP_NAME_CASES) { + checks.push({ + id: c.id, + name: c.name, + description: c.description, + status: 'SKIPPED', + timestamp, + errorMessage: reason, + specReferences: [c.specRef], + details: { skippedReason: 'requires-fixture' } + }); + } +} + /** * Helper to send a raw HTTP POST request with custom headers. * Uses Node.js http.request to preserve exact header casing and values, @@ -292,6 +309,7 @@ function createAcceptanceCheck( export class HttpHeaderValidationScenario implements ClientScenario { name = 'http-header-validation'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'] } as const; description = `Test server validation of standard MCP request headers (SEP-2243). **Server Implementation Requirements:** @@ -383,7 +401,12 @@ export class HttpHeaderValidationScenario implements ClientScenario { { requestBodyMethod: 'tools/list', mcpMethodHeader: '(missing)' } ); - if (toolsResult.tools && toolsResult.tools.length > 0) { + if (ctx.production) { + skippedMcpNameCases( + checks, + 'Skipped in production mode: Mcp-Name tools/call acceptance check is not invoked against arbitrary production tools' + ); + } else if (toolsResult.tools && toolsResult.tools.length > 0) { const toolName = toolsResult.tools[0].name; await this.testCase( @@ -603,6 +626,7 @@ export class HttpHeaderValidationScenario implements ClientScenario { export class HttpCustomHeaderServerValidationScenario implements ClientScenario { name = 'http-custom-header-server-validation'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test server validation of custom Mcp-Param headers and Base64 encoding (SEP-2243). **Server Implementation Requirements:** diff --git a/src/scenarios/server/input-required-result.ts b/src/scenarios/server/input-required-result.ts index 64ad4ba9..21f7a54e 100644 --- a/src/scenarios/server/input-required-result.ts +++ b/src/scenarios/server/input-required-result.ts @@ -29,6 +29,7 @@ import { notTestable } from '../untestable'; export class InputRequiredResultBasicElicitationScenario implements ClientScenario { name = 'input-required-result-basic-elicitation'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test basic ephemeral InputRequiredResult flow with a single elicitation input request (SEP-2322). @@ -184,6 +185,7 @@ Implement a tool named \`test_input_required_result_elicitation\` (no arguments export class InputRequiredResultBasicSamplingScenario implements ClientScenario { name = 'input-required-result-basic-sampling'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test basic ephemeral InputRequiredResult flow with a single sampling input request (SEP-2322). @@ -326,6 +328,7 @@ Implement a tool named \`test_input_required_result_sampling\` (no arguments req export class InputRequiredResultBasicListRootsScenario implements ClientScenario { name = 'input-required-result-basic-list-roots'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test basic ephemeral InputRequiredResult flow with a single roots/list input request (SEP-2322). @@ -462,6 +465,7 @@ Implement a tool named \`test_input_required_result_list_roots\` (no arguments r export class InputRequiredResultRequestStateScenario implements ClientScenario { name = 'input-required-result-request-state'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that requestState is correctly round-tripped in ephemeral InputRequiredResult flow (SEP-2322). @@ -594,6 +598,7 @@ Implement a tool named \`test_input_required_result_request_state\` (no argument export class InputRequiredResultMultipleInputRequestsScenario implements ClientScenario { name = 'input-required-result-multiple-input-requests'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test multiple input requests in a single InputRequiredResult (SEP-2322). @@ -769,6 +774,7 @@ Implement a tool named \`test_input_required_result_multiple_inputs\` (no argume export class InputRequiredResultMultiRoundScenario implements ClientScenario { name = 'input-required-result-multi-round'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test multi-round ephemeral InputRequiredResult flow with evolving requestState (SEP-2322). @@ -954,6 +960,7 @@ Implement a tool named \`test_input_required_result_multi_round\` (no arguments export class InputRequiredResultMissingInputResponseScenario implements ClientScenario { name = 'input-required-result-missing-input-response'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test error handling when client sends wrong/missing inputResponses (SEP-2322). @@ -1028,6 +1035,7 @@ Use the same tool as A1: \`test_input_required_result_elicitation\`. export class InputRequiredResultNonToolRequestScenario implements ClientScenario { name = 'input-required-result-non-tool-request'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['prompts'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test InputRequiredResult on a non-tool request (prompts/get) to verify InputRequiredResult is universal (SEP-2322). @@ -1153,6 +1161,7 @@ Implement a prompt named \`test_input_required_result_prompt\` that requires eli export class InputRequiredResultResultTypeScenario implements ClientScenario { name = 'input-required-result-result-type'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server explicitly includes resultType field in InputRequiredResult responses (SEP-2322). @@ -1222,6 +1231,7 @@ This scenario verifies that the resultType field is explicitly present in the re export class InputRequiredResultUnsupportedMethodsScenario implements ClientScenario { name = 'input-required-result-unsupported-methods'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { anyCapability: ['tools', 'prompts'] } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server does NOT return InputRequiredResult on unsupported methods (SEP-2322). @@ -1280,6 +1290,7 @@ Servers MUST NOT send InputRequiredResult responses on any client requests other export class InputRequiredResultTamperedStateScenario implements ClientScenario { name = 'input-required-result-tampered-state'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server rejects tampered requestState (SEP-2322). @@ -1395,6 +1406,7 @@ JSON-RPC error (code -32602 or similar) indicating integrity check failure.`; export class InputRequiredResultCapabilityCheckScenario implements ClientScenario { name = 'input-required-result-capability-check'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server only sends inputRequests for capabilities the client declared (SEP-2322). @@ -1493,6 +1505,7 @@ Only include inputRequests for methods the client supports. For example, if the export class InputRequiredResultIgnoreExtraParamsScenario implements ClientScenario { name = 'input-required-result-ignore-extra-params'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server ignores unexpected extra parameters in InputResponses (SEP-2322). @@ -1567,6 +1580,7 @@ the extra keys and complete normally.`; export class InputRequiredResultValidateInputScenario implements ClientScenario { name = 'input-required-result-validate-input'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION]; description = `Test that server validates InputResponses and returns appropriate errors (SEP-2322). diff --git a/src/scenarios/server/json-schema-2020-12.ts b/src/scenarios/server/json-schema-2020-12.ts index d1487a5e..65e48345 100644 --- a/src/scenarios/server/json-schema-2020-12.ts +++ b/src/scenarios/server/json-schema-2020-12.ts @@ -88,6 +88,7 @@ export function sep2106KeywordCheckStatus( export class JsonSchema2020_12Scenario implements ClientScenario { name = 'json-schema-2020-12'; readonly source = { introducedIn: '2025-11-25' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Validates JSON Schema 2020-12 keyword preservation (SEP-1613, SEP-2106). **Server Implementation Requirements:** diff --git a/src/scenarios/server/production-mode-fixtures.ts b/src/scenarios/server/production-mode-fixtures.ts new file mode 100644 index 00000000..a3fe9c6a --- /dev/null +++ b/src/scenarios/server/production-mode-fixtures.ts @@ -0,0 +1,330 @@ +/** + * Minimal loopback MCP server fixtures for `--production` mode integration + * tests (stateful 2025-11-25 and stateless 2026-07-28). + */ +import http from 'http'; +import path from 'path'; +import { spawn } from 'child_process'; +import { fileURLToPath } from 'url'; +import type { AddressInfo } from 'net'; +import { DRAFT_PROTOCOL_VERSION, LATEST_SPEC_VERSION } from '../../types'; + +const REPO_ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..' +); + +export interface TestServer { + url: string; + methodsCalled: string[]; + close: () => Promise; +} + +export interface StatefulServerConfig { + capabilities: Record; + failResourcesList?: boolean; +} + +export interface StatelessServerConfig { + capabilities: Record; + includeCachingHints: boolean; +} + +const DOMAIN_TOOLS = [ + { + name: 'execute_sql', + description: 'Execute a read-only SQL query', + inputSchema: { + type: 'object', + properties: { sql: { type: 'string' } }, + required: ['sql'] + } + }, + { + name: 'list_tables', + description: 'List database tables', + inputSchema: { type: 'object', properties: {} } + } +]; + +function isLoopbackHost(hostHeader: string | undefined): boolean { + if (!hostHeader) return false; + const host = hostHeader.split(':')[0]; + return host === '127.0.0.1' || host === 'localhost' || host === '[::1]'; +} + +function isAllowedOrigin(originHeader: string | undefined): boolean { + if (!originHeader) return true; + try { + const { hostname } = new URL(originHeader); + return ( + hostname === '127.0.0.1' || + hostname === 'localhost' || + hostname === '[::1]' + ); + } catch { + return false; + } +} + +function sendJson( + res: http.ServerResponse, + status: number, + payload: unknown, + headers: Record = {} +): void { + res.writeHead(status, { 'Content-Type': 'application/json', ...headers }); + res.end(JSON.stringify(payload)); +} + +function readJsonBody( + req: http.IncomingMessage, + onBody: (body: Record) => void +): void { + let raw = ''; + req.on('data', (chunk) => { + raw += chunk; + }); + req.on('end', () => { + onBody(JSON.parse(raw) as Record); + }); +} + +async function listenLoopback( + handler: http.RequestListener, + methodsCalled: string[] +): Promise { + const server = http.createServer(handler); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', () => resolve()) + ); + const port = (server.address() as AddressInfo).port; + return { + url: `http://127.0.0.1:${port}/mcp`, + methodsCalled, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }) + }; +} + +function dispatchStatefulMethod( + res: http.ServerResponse, + id: unknown, + method: string, + config: StatefulServerConfig +): void { + if (method === 'ping') { + sendJson(res, 200, { jsonrpc: '2.0', id, result: {} }); + return; + } + if (method === 'tools/list') { + sendJson(res, 200, { + jsonrpc: '2.0', + id, + result: { tools: DOMAIN_TOOLS } + }); + return; + } + if ( + method === 'resources/list' && + 'resources' in config.capabilities && + !config.failResourcesList + ) { + sendJson(res, 200, { jsonrpc: '2.0', id, result: { resources: [] } }); + return; + } + sendJson(res, 200, { + jsonrpc: '2.0', + id, + error: { code: -32601, message: `Method not supported: ${method}` } + }); +} + +function handleStatefulRpc( + req: http.IncomingMessage, + res: http.ServerResponse, + body: Record, + config: StatefulServerConfig, + activeSessions: Set, + nextSessionId: () => string +): void { + const id = body.id ?? null; + const method = String(body.method ?? ''); + if (id === null || method.startsWith('notifications/')) { + res.writeHead(202).end(); + return; + } + if (method === 'initialize') { + const sessionId = nextSessionId(); + activeSessions.add(sessionId); + const params = (body.params as Record) ?? {}; + const requestedVersion = + typeof params.protocolVersion === 'string' + ? params.protocolVersion + : LATEST_SPEC_VERSION; + sendJson( + res, + 200, + { + jsonrpc: '2.0', + id, + result: { + protocolVersion: requestedVersion, + capabilities: config.capabilities, + serverInfo: { name: 'dbhub-like-prod-server', version: '1.0.0' } + } + }, + { 'mcp-session-id': sessionId } + ); + return; + } + + const sessionId = String(req.headers['mcp-session-id'] ?? ''); + if (!sessionId || !activeSessions.has(sessionId)) { + sendJson(res, 404, { + jsonrpc: '2.0', + id, + error: { code: -32001, message: 'Session not found' } + }); + return; + } + dispatchStatefulMethod(res, id, method, config); +} + +export async function startStatefulPartialServer( + config: StatefulServerConfig +): Promise { + const methodsCalled: string[] = []; + const activeSessions = new Set(); + let sessionCounter = 0; + const nextSessionId = () => `prod-session-${++sessionCounter}`; + + return listenLoopback((req, res) => { + if ( + !isLoopbackHost(req.headers.host) || + !isAllowedOrigin(req.headers.origin) + ) { + sendJson(res, 403, { error: 'Forbidden host or origin' }); + return; + } + if (req.method === 'DELETE') { + const sid = String(req.headers['mcp-session-id'] ?? ''); + res.writeHead(sid && activeSessions.delete(sid) ? 200 : 404).end(); + return; + } + if (req.method === 'GET') { + res.writeHead(405).end(); + return; + } + readJsonBody(req, (body) => { + methodsCalled.push(String(body.method ?? '')); + handleStatefulRpc(req, res, body, config, activeSessions, nextSessionId); + }); + }, methodsCalled); +} + +function handleStatelessRpc( + req: http.IncomingMessage, + res: http.ServerResponse, + body: Record, + config: StatelessServerConfig +): void { + const id = body.id ?? null; + const method = String(body.method ?? ''); + const mcpMethod = req.headers['mcp-method']; + if (typeof mcpMethod !== 'string' || mcpMethod.trim() !== method) { + sendJson(res, 400, { + jsonrpc: '2.0', + id, + error: { code: -32020, message: 'Mcp-Method header mismatch' } + }); + return; + } + if (method === 'server/discover') { + sendJson(res, 200, { + jsonrpc: '2.0', + id, + result: { + resultType: 'complete', + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: config.capabilities, + ttlMs: 30000, + cacheScope: 'public', + _meta: { + 'io.modelcontextprotocol/serverInfo': { + name: 'stateless-tools-only', + version: '1.0.0' + } + } + } + }); + return; + } + if (method === 'tools/list' && 'tools' in config.capabilities) { + const result = config.includeCachingHints + ? { + resultType: 'complete', + tools: DOMAIN_TOOLS, + ttlMs: 30000, + cacheScope: 'public' + } + : { resultType: 'complete', tools: DOMAIN_TOOLS }; + sendJson(res, 200, { jsonrpc: '2.0', id, result }); + return; + } + sendJson(res, 404, { + jsonrpc: '2.0', + id, + error: { code: -32601, message: `Method not found: ${method}` } + }); +} + +export async function startStatelessPartialServer( + config: StatelessServerConfig +): Promise { + const methodsCalled: string[] = []; + return listenLoopback((req, res) => { + if ( + !isLoopbackHost(req.headers.host) || + !isAllowedOrigin(req.headers.origin) + ) { + sendJson(res, 403, { error: 'Forbidden host or origin' }); + return; + } + readJsonBody(req, (body) => { + methodsCalled.push(String(body.method ?? '')); + handleStatelessRpc(req, res, body, config); + }); + }, methodsCalled); +} + +export function runCliSubprocess( + args: string[] +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn( + process.execPath, + [ + path.join(REPO_ROOT, 'node_modules/tsx/dist/cli.mjs'), + path.join(REPO_ROOT, 'src/index.ts'), + ...args + ], + { cwd: REPO_ROOT, stdio: ['ignore', 'pipe', 'pipe'] } + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.on('error', reject); + child.on('close', (code) => + resolve({ exitCode: code ?? 1, stdout, stderr }) + ); + }); +} diff --git a/src/scenarios/server/production-mode.test.ts b/src/scenarios/server/production-mode.test.ts new file mode 100644 index 00000000..8ef665b3 --- /dev/null +++ b/src/scenarios/server/production-mode.test.ts @@ -0,0 +1,354 @@ +/** + * Integration tests for server conformance `--production` mode and SEP-2549 + * capability-aware caching checks (issue #516). + */ +import { afterEach, describe, expect, it } from 'vitest'; +import { + printServerSummary, + runServerConformanceTest +} from '../../runner/server'; +import { listActiveClientScenarios } from '../index'; +import { + DRAFT_PROTOCOL_VERSION, + type ConformanceCheck, + type SpecVersion +} from '../../types'; +import { + runCliSubprocess, + startStatefulPartialServer, + startStatelessPartialServer, + type TestServer +} from './production-mode-fixtures'; + +describe('server conformance --production mode (stateful 2025-11-25)', () => { + let activeServer: TestServer | undefined; + + afterEach(async () => { + if (activeServer) { + await activeServer.close(); + activeServer = undefined; + } + }); + + it('preserves strict SDK failures in default mode when capabilities or fixtures are missing', async () => { + activeServer = await startStatefulPartialServer({ + capabilities: { tools: {} } + }); + + const missingCap = await runServerConformanceTest( + activeServer.url, + 'resources-list' + ); + expect(missingCap.checks.some((c) => c.status === 'FAILURE')).toBe(true); + + const missingFixture = await runServerConformanceTest( + activeServer.url, + 'tools-call-simple-text' + ); + expect(missingFixture.checks.some((c) => c.status === 'FAILURE')).toBe( + true + ); + }); + + it('passes active suite against a healthy tools-only production server and skips inapplicable scenarios', async () => { + activeServer = await startStatefulPartialServer({ + capabilities: { tools: {} } + }); + + const scenarios = listActiveClientScenarios(); + const capabilitiesCache = new Map>(); + const allResults: Array<{ scenario: string; checks: ConformanceCheck[] }> = + []; + + for (const scenario of scenarios) { + const result = await runServerConformanceTest( + activeServer.url, + scenario, + undefined, + undefined, + false, + 10000, + { production: true, capabilitiesCache } + ); + allResults.push({ scenario, checks: result.checks }); + } + + const summary = printServerSummary(allResults); + expect(summary.totalFailed).toBe(0); + expect(summary.totalPassed).toBeGreaterThan(0); + expect(summary.totalSkipped).toBeGreaterThan(0); + + const byScenario = new Map(allResults.map((r) => [r.scenario, r.checks])); + for (const executed of [ + 'server-initialize', + 'server-session-lifecycle', + 'ping', + 'dns-rebinding-protection', + 'tools-list', + 'server-sse-multiple-streams' + ]) { + const checks = byScenario.get(executed) ?? []; + expect( + checks.some((c) => c.status === 'SUCCESS'), + executed + ).toBe(true); + expect( + checks.some((c) => c.status === 'FAILURE'), + executed + ).toBe(false); + } + + expect(byScenario.get('resources-list')?.[0]).toMatchObject({ + status: 'SKIPPED', + details: { + skippedReason: 'missing-capability', + missingCapabilities: ['resources'] + } + }); + expect(byScenario.get('tools-call-simple-text')?.[0]).toMatchObject({ + status: 'SKIPPED', + details: { skippedReason: 'requires-fixture' } + }); + }, 30000); + + it('still fails in production mode when a server declares resources but fails resources/list', async () => { + activeServer = await startStatefulPartialServer({ + capabilities: { tools: {}, resources: {} }, + failResourcesList: true + }); + + const result = await runServerConformanceTest( + activeServer.url, + 'resources-list', + undefined, + undefined, + false, + 10000, + { production: true } + ); + expect(result.checks.some((c) => c.status === 'FAILURE')).toBe(true); + }); + + it('never falsely skips an unreachable server in production mode', async () => { + const result = await runServerConformanceTest( + 'http://127.0.0.1:9/mcp', + 'tools-list', + undefined, + undefined, + false, + 5000, + { production: true } + ); + expect(result.checks.some((c) => c.status === 'FAILURE')).toBe(true); + expect(result.checks.every((c) => c.status === 'SKIPPED')).toBe(false); + }); +}); + +describe('SEP-2549 caching and stateless production mode (2026-07-28)', () => { + let activeServer: TestServer | undefined; + + afterEach(async () => { + if (activeServer) { + await activeServer.close(); + activeServer = undefined; + } + }); + + it('queries only declared capabilities in caching and skips undeclared endpoints', async () => { + activeServer = await startStatelessPartialServer({ + capabilities: { tools: {} }, + includeCachingHints: true + }); + + const result = await runServerConformanceTest( + activeServer.url, + 'caching', + undefined, + DRAFT_PROTOCOL_VERSION, + false, + 10000, + { production: true } + ); + + expect(activeServer.methodsCalled).toContain('tools/list'); + expect(activeServer.methodsCalled).not.toContain('prompts/list'); + expect(activeServer.methodsCalled).not.toContain('resources/list'); + expect(activeServer.methodsCalled).not.toContain( + 'resources/templates/list' + ); + expect(activeServer.methodsCalled).not.toContain('resources/read'); + + const statusById = new Map(result.checks.map((c) => [c.id, c.status])); + expect(statusById.get('sep-2549-tools-list-caching-hints')).toBe('SUCCESS'); + expect(statusById.get('sep-2549-ttl-non-negative')).toBe('SUCCESS'); + expect(statusById.get('sep-2549-cache-scope-valid')).toBe('SUCCESS'); + expect(statusById.get('sep-2549-prompts-list-caching-hints')).toBe( + 'SKIPPED' + ); + expect(statusById.get('sep-2549-resources-list-caching-hints')).toBe( + 'SKIPPED' + ); + expect( + statusById.get('sep-2549-resources-templates-list-caching-hints') + ).toBe('SKIPPED'); + expect(statusById.get('sep-2549-resources-read-caching-hints')).toBe( + 'SKIPPED' + ); + }); + + it('fails caching in production mode when declared tools/list omits caching hints', async () => { + activeServer = await startStatelessPartialServer({ + capabilities: { tools: {} }, + includeCachingHints: false + }); + + const result = await runServerConformanceTest( + activeServer.url, + 'caching', + undefined, + DRAFT_PROTOCOL_VERSION, + false, + 10000, + { production: true } + ); + expect(result.checks.some((c) => c.status === 'FAILURE')).toBe(true); + }); + + it('fails empty-capability server on caching in default mode and skips in production mode', async () => { + activeServer = await startStatelessPartialServer({ + capabilities: {}, + includeCachingHints: false + }); + + const strictResult = await runServerConformanceTest( + activeServer.url, + 'caching', + undefined, + DRAFT_PROTOCOL_VERSION + ); + expect(strictResult.checks.some((c) => c.status === 'FAILURE')).toBe(true); + + const prodResult = await runServerConformanceTest( + activeServer.url, + 'caching', + undefined, + DRAFT_PROTOCOL_VERSION, + false, + 10000, + { production: true } + ); + expect(prodResult.checks).toHaveLength(1); + expect(prodResult.checks[0]).toMatchObject({ + id: 'caching', + status: 'SKIPPED', + details: { + skippedReason: 'missing-capability', + requiredAnyOf: ['tools', 'prompts', 'resources'] + } + }); + }); + + it('runs http-header-validation without invoking tools/call on domain tools in production mode', async () => { + activeServer = await startStatelessPartialServer({ + capabilities: { tools: {} }, + includeCachingHints: true + }); + + const result = await runServerConformanceTest( + activeServer.url, + 'http-header-validation', + undefined, + DRAFT_PROTOCOL_VERSION, + false, + 10000, + { production: true } + ); + + expect(activeServer.methodsCalled).not.toContain('tools/call'); + expect(result.checks.some((c) => c.status === 'FAILURE')).toBe(false); + expect(result.checks.some((c) => c.status === 'SUCCESS')).toBe(true); + expect(result.checks.some((c) => c.status === 'SKIPPED')).toBe(true); + }); + + it('skips missing diagnostic fixture probes in server-stateless under production mode', async () => { + activeServer = await startStatelessPartialServer({ + capabilities: { tools: {} }, + includeCachingHints: true + }); + + const result = await runServerConformanceTest( + activeServer.url, + 'server-stateless', + undefined, + DRAFT_PROTOCOL_VERSION, + false, + 15000, + { production: true } + ); + + const byId = new Map(result.checks.map((c) => [c.id, c])); + expect(byId.get('sep-2575-server-implements-discover')?.status).toBe( + 'SUCCESS' + ); + expect( + byId.get('sep-2575-server-rejects-undeclared-capability') + ).toMatchObject({ + status: 'SKIPPED', + details: { + skippedReason: 'requires-fixture', + fixture: 'test_missing_capability' + } + }); + expect(byId.get('sep-2575-missing-capability-http-400')?.status).toBe( + 'SKIPPED' + ); + }, 30000); +}); + +describe('CLI --production flag', () => { + let activeServer: TestServer | undefined; + + afterEach(async () => { + if (activeServer) { + await activeServer.close(); + activeServer = undefined; + } + }); + + it('exits 1 without --production and exits 0 with --production on a capability-partial server', async () => { + activeServer = await startStatefulPartialServer({ + capabilities: { tools: {} } + }); + + const defaultRun = await runCliSubprocess([ + 'server', + '--url', + activeServer.url, + '--scenario', + 'resources-list' + ]); + expect(defaultRun.exitCode).toBe(1); + expect(defaultRun.stdout).toContain('Failed Checks'); + + const prodScenarioRun = await runCliSubprocess([ + 'server', + '--url', + activeServer.url, + '--scenario', + 'resources-list', + '--production' + ]); + expect(prodScenarioRun.exitCode).toBe(0); + expect(prodScenarioRun.stdout).toContain('1 skipped'); + + const prodSuiteRun = await runCliSubprocess([ + 'server', + '--url', + activeServer.url, + '--production' + ]); + expect(prodSuiteRun.exitCode).toBe(0); + expect(prodSuiteRun.stdout).toContain('0 failed'); + expect(prodSuiteRun.stdout).toContain('skipped'); + }, 60000); +}); diff --git a/src/scenarios/server/prompts.ts b/src/scenarios/server/prompts.ts index c5a5f380..f1ff21f0 100644 --- a/src/scenarios/server/prompts.ts +++ b/src/scenarios/server/prompts.ts @@ -12,6 +12,7 @@ import type { export class PromptsListScenario implements ClientScenario { name = 'prompts-list'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['prompts'] } as const; description = `Test listing available prompts. **Server Implementation Requirements:** @@ -93,6 +94,7 @@ export class PromptsListScenario implements ClientScenario { export class PromptsGetSimpleScenario implements ClientScenario { name = 'prompts-get-simple'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['prompts'], fixture: true } as const; description = `Test getting a simple prompt without arguments. **Server Implementation Requirements:** @@ -178,6 +180,7 @@ Implement a prompt named \`test_simple_prompt\` with no arguments that returns: export class PromptsGetWithArgsScenario implements ClientScenario { name = 'prompts-get-with-args'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['prompts'], fixture: true } as const; description = `Test parameterized prompt. **Server Implementation Requirements:** @@ -274,6 +277,7 @@ Returns (with args \`{arg1: "hello", arg2: "world"}\`): export class PromptsGetEmbeddedResourceScenario implements ClientScenario { name = 'prompts-get-embedded-resource'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['prompts'], fixture: true } as const; description = `Test prompt with embedded resource content. **Server Implementation Requirements:** @@ -380,6 +384,7 @@ Returns: export class PromptsGetWithImageScenario implements ClientScenario { name = 'prompts-get-with-image'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['prompts'], fixture: true } as const; description = `Test prompt with image content. **Server Implementation Requirements:** diff --git a/src/scenarios/server/requirements-check.test.ts b/src/scenarios/server/requirements-check.test.ts new file mode 100644 index 00000000..1965c18a --- /dev/null +++ b/src/scenarios/server/requirements-check.test.ts @@ -0,0 +1,227 @@ +import { describe, it, expect } from 'vitest'; +import { + hasServerCapability, + hasServerExtension, + evaluateScenarioRequirements +} from './requirements-check'; +import { getClientScenario, listClientScenarios } from '../index'; + +describe('hasServerCapability', () => { + it('matches top-level server capabilities when declared', () => { + const caps = { + tools: {}, + resources: { listChanged: true }, + prompts: {}, + logging: {}, + completions: {} + }; + + expect(hasServerCapability(caps, 'tools')).toBe(true); + expect(hasServerCapability(caps, 'resources')).toBe(true); + expect(hasServerCapability(caps, 'prompts')).toBe(true); + expect(hasServerCapability(caps, 'logging')).toBe(true); + expect(hasServerCapability(caps, 'completions')).toBe(true); + }); + + it('returns false when top-level capabilities are absent or undefined', () => { + expect(hasServerCapability({}, 'tools')).toBe(false); + expect(hasServerCapability({ resources: undefined }, 'resources')).toBe( + false + ); + }); + + it('treats malformed non-undefined declarations as declared so validation runs', () => { + expect(hasServerCapability({ resources: true }, 'resources')).toBe(true); + expect(hasServerCapability({ tools: null }, 'tools')).toBe(true); + }); + + it('checks nested resources.subscribe capability flag', () => { + expect(hasServerCapability({}, 'resources.subscribe')).toBe(false); + expect(hasServerCapability({ resources: {} }, 'resources.subscribe')).toBe( + false + ); + expect( + hasServerCapability( + { resources: { subscribe: false } }, + 'resources.subscribe' + ) + ).toBe(false); + expect( + hasServerCapability( + { resources: { subscribe: true } }, + 'resources.subscribe' + ) + ).toBe(true); + }); +}); + +describe('hasServerExtension', () => { + it('detects declared extensions under capabilities.extensions', () => { + expect(hasServerExtension({}, 'io.modelcontextprotocol/tasks')).toBe(false); + expect( + hasServerExtension({ extensions: {} }, 'io.modelcontextprotocol/tasks') + ).toBe(false); + expect( + hasServerExtension( + { extensions: { 'io.modelcontextprotocol/tasks': {} } }, + 'io.modelcontextprotocol/tasks' + ) + ).toBe(true); + expect( + hasServerExtension( + { extensions: { 'io.modelcontextprotocol/skills': true } }, + 'io.modelcontextprotocol/skills' + ) + ).toBe(true); + }); +}); + +describe('evaluateScenarioRequirements', () => { + it('returns null when scenario has no requirements', () => { + expect( + evaluateScenarioRequirements('server-initialize', undefined, {}) + ).toBeNull(); + }); + + it('skips when required capabilities are missing', () => { + const check = evaluateScenarioRequirements( + 'resources-list', + { capabilities: ['resources'] }, + { tools: {} } + ); + expect(check).toMatchObject({ + id: 'resources-list', + status: 'SKIPPED', + details: { + skippedReason: 'missing-capability', + missingCapabilities: ['resources'] + } + }); + }); + + it('evaluates anyCapability with OR semantics', () => { + const skipped = evaluateScenarioRequirements( + 'caching', + { anyCapability: ['tools', 'prompts', 'resources'] }, + { logging: {} } + ); + expect(skipped).toMatchObject({ + id: 'caching', + status: 'SKIPPED', + details: { + skippedReason: 'missing-capability', + requiredAnyOf: ['tools', 'prompts', 'resources'] + } + }); + + const allowed = evaluateScenarioRequirements( + 'caching', + { anyCapability: ['tools', 'prompts', 'resources'] }, + { tools: {} } + ); + expect(allowed).toBeNull(); + }); + + it('skips when required extensions are missing', () => { + const check = evaluateScenarioRequirements( + 'sep-2640-skills-manifest', + { extensions: ['io.modelcontextprotocol/skills'] }, + { tools: {} } + ); + expect(check).toMatchObject({ + id: 'sep-2640-skills-manifest', + status: 'SKIPPED', + details: { + skippedReason: 'missing-extension', + missingExtensions: ['io.modelcontextprotocol/skills'] + } + }); + }); + + it('skips fixture-bound scenarios after capabilities are satisfied', () => { + const missingCapFirst = evaluateScenarioRequirements( + 'tools-call-simple-text', + { capabilities: ['tools'], fixture: true }, + {} + ); + expect(missingCapFirst?.details?.skippedReason).toBe('missing-capability'); + + const fixtureSkip = evaluateScenarioRequirements( + 'tools-call-simple-text', + { capabilities: ['tools'], fixture: true }, + { tools: {} } + ); + expect(fixtureSkip).toMatchObject({ + id: 'tools-call-simple-text', + status: 'SKIPPED', + details: { + skippedReason: 'requires-fixture', + scenario: 'tools-call-simple-text' + } + }); + }); +}); + +describe('server scenario requires metadata inventory', () => { + it('keeps base protocol scenarios free of optional capability and fixture gates', () => { + const baseProtocolScenarios = [ + 'server-initialize', + 'server-session-lifecycle', + 'ping', + 'dns-rebinding-protection', + 'server-stateless' + ]; + for (const name of baseProtocolScenarios) { + const scenario = getClientScenario(name); + expect(scenario, name).toBeDefined(); + expect(scenario?.requires, name).toBeUndefined(); + } + }); + + it('marks generic capability scenarios without fixture: true', () => { + const genericCapabilityScenarios: Array<[string, string[]]> = [ + ['tools-list', ['tools']], + ['resources-list', ['resources']], + ['sep-2164-resource-not-found', ['resources']], + ['prompts-list', ['prompts']], + ['logging-set-level', ['logging']], + ['server-sse-multiple-streams', ['tools']], + ['http-header-validation', ['tools']] + ]; + for (const [name, expectedCaps] of genericCapabilityScenarios) { + const scenario = getClientScenario(name); + expect(scenario?.requires?.capabilities, name).toEqual(expectedCaps); + expect(scenario?.requires?.fixture, name).toBeUndefined(); + } + + const caching = getClientScenario('caching'); + expect(caching?.requires).toEqual({ + anyCapability: ['tools', 'prompts', 'resources'] + }); + }); + + it('marks reference-fixture scenarios with fixture: true', () => { + const allNames = listClientScenarios(); + const fixturePrefixes = [ + 'tools-call-', + 'prompts-get-', + 'resources-read-', + 'resources-templates-read', + 'resources-subscribe', + 'resources-unsubscribe', + 'completion-complete', + 'json-schema-2020-12', + 'elicitation-sep', + 'server-sse-polling', + 'http-custom-header-server-validation', + 'tasks-' + ]; + + for (const name of allNames) { + if (fixturePrefixes.some((prefix) => name.startsWith(prefix))) { + const scenario = getClientScenario(name); + expect(scenario?.requires?.fixture, name).toBe(true); + } + } + }); +}); diff --git a/src/scenarios/server/requirements-check.ts b/src/scenarios/server/requirements-check.ts new file mode 100644 index 00000000..44756a30 --- /dev/null +++ b/src/scenarios/server/requirements-check.ts @@ -0,0 +1,154 @@ +/** + * Capability and fixture requirement evaluation for server-conformance scenarios. + * + * In default SDK conformance mode (`--production` omitted), scenarios execute + * unconditionally so SDK reference servers cannot pass by omitting features + * (issues #248, #372). In opt-in `--production` mode, the runner evaluates + * each scenario's declarative `requires` metadata against the server's + * advertised capabilities and emits `SKIPPED` checks for undeclared optional + * capabilities/extensions or reference-fixture contracts. + */ + +import type { + ConformanceCheck, + ExtensionId, + ScenarioRequirements, + ServerCapability +} from '../../types'; + +export type SkippedRequirementReason = + | 'missing-capability' + | 'missing-extension' + | 'requires-fixture'; + +export function isPlainObject( + value: unknown +): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * Whether `capabilities` declares `capability`. + * + * Checks key presence (`!== undefined`) for top-level capability objects so a + * malformed declaration (e.g. `resources: true`) is still treated as declared + * and allowed to fail wire/scenario validation rather than silently skipping. + */ +export function hasServerCapability( + capabilities: Record, + capability: ServerCapability +): boolean { + if (capability === 'resources.subscribe') { + return ( + isPlainObject(capabilities.resources) && + Boolean(capabilities.resources.subscribe) + ); + } + return capability in capabilities && capabilities[capability] !== undefined; +} + +/** + * Whether `capabilities.extensions` declares `extensionId`. + */ +export function hasServerExtension( + capabilities: Record, + extensionId: ExtensionId +): boolean { + if (!isPlainObject(capabilities.extensions)) { + return false; + } + return ( + extensionId in capabilities.extensions && + capabilities.extensions[extensionId] !== undefined + ); +} + +function buildSkippedCheck( + scenarioName: string, + description: string, + skippedReason: SkippedRequirementReason, + extraDetails: Record +): ConformanceCheck { + return { + id: scenarioName, + name: scenarioName, + description, + status: 'SKIPPED', + timestamp: new Date().toISOString(), + details: { + skippedReason, + ...extraDetails + } + }; +} + +function formatQuotedList(items: readonly string[]): string { + return items.map((item) => `'${item}'`).join(', '); +} + +/** + * Evaluate a scenario's `requires` metadata against the server's declared + * `capabilities` in `--production` mode. + * + * Returns a `SKIPPED` check when a prerequisite is unmet, or `null` when all + * prerequisites are satisfied and the scenario should execute. + */ +export function evaluateScenarioRequirements( + scenarioName: string, + requires: ScenarioRequirements | undefined, + capabilities: Record +): ConformanceCheck | null { + if (!requires) { + return null; + } + + const missingCaps = (requires.capabilities ?? []).filter( + (cap) => !hasServerCapability(capabilities, cap) + ); + if (missingCaps.length > 0) { + return buildSkippedCheck( + scenarioName, + `Skipped in production mode: server did not declare required capability ${formatQuotedList(missingCaps)}`, + 'missing-capability', + { missingCapabilities: missingCaps } + ); + } + + if ( + requires.anyCapability && + requires.anyCapability.length > 0 && + !requires.anyCapability.some((cap) => + hasServerCapability(capabilities, cap) + ) + ) { + return buildSkippedCheck( + scenarioName, + `Skipped in production mode: server did not declare any of ${formatQuotedList(requires.anyCapability)}`, + 'missing-capability', + { requiredAnyOf: [...requires.anyCapability] } + ); + } + + const missingExts = (requires.extensions ?? []).filter( + (ext) => !hasServerExtension(capabilities, ext) + ); + if (missingExts.length > 0) { + return buildSkippedCheck( + scenarioName, + `Skipped in production mode: server did not declare required extension ${formatQuotedList(missingExts)}`, + 'missing-extension', + { missingExtensions: missingExts } + ); + } + + if (requires.fixture) { + return buildSkippedCheck( + scenarioName, + `Skipped in production mode: scenario '${scenarioName}' requires reference-fixture artifacts (test_* / test://)`, + 'requires-fixture', + { scenario: scenarioName } + ); + } + + return null; +} diff --git a/src/scenarios/server/resources.ts b/src/scenarios/server/resources.ts index 382b0db8..21b01430 100644 --- a/src/scenarios/server/resources.ts +++ b/src/scenarios/server/resources.ts @@ -19,6 +19,7 @@ import type { export class ResourcesListScenario implements ClientScenario { name = 'resources-list'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['resources'] } as const; description = `Test listing available resources. **Server Implementation Requirements:** @@ -100,6 +101,7 @@ export class ResourcesListScenario implements ClientScenario { export class ResourcesReadTextScenario implements ClientScenario { name = 'resources-read-text'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['resources'], fixture: true } as const; description = `Test reading text resource. **Server Implementation Requirements:** @@ -187,6 +189,7 @@ Implement resource \`test://static-text\` that returns: export class ResourcesReadBinaryScenario implements ClientScenario { name = 'resources-read-binary'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['resources'], fixture: true } as const; description = `Test reading binary resource. **Server Implementation Requirements:** @@ -272,6 +275,7 @@ Implement resource \`test://static-binary\` that returns: export class ResourcesTemplateReadScenario implements ClientScenario { name = 'resources-templates-read'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['resources'], fixture: true } as const; description = `Test reading resource from template. **Server Implementation Requirements:** @@ -377,6 +381,10 @@ export class ResourcesSubscribeScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { + capabilities: ['resources.subscribe'], + fixture: true + } as const; description = `Test subscribing to resource updates. **Server Implementation Requirements:** @@ -448,6 +456,7 @@ Example request: export class ResourcesNotFoundErrorScenario implements ClientScenario { name = 'sep-2164-resource-not-found'; readonly source = { introducedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['resources'] } as const; description = `Test error handling for non-existent resources (SEP-2164). **Server Implementation Requirements:** @@ -607,6 +616,10 @@ export class ResourcesUnsubscribeScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { + capabilities: ['resources.subscribe'], + fixture: true + } as const; description = `Test unsubscribing from resource. **Server Implementation Requirements:** diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts index 91dae452..540efd13 100644 --- a/src/scenarios/server/skills/directory.ts +++ b/src/scenarios/server/skills/directory.ts @@ -124,6 +124,7 @@ async function discoverDirectory( export class SkillsDirectoryReadScenario implements ClientScenario { name = 'sep-2640-skills-directory'; readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + readonly requires = { extensions: [SKILLS_EXTENSION_ID] } as const; description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). **Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts index 68b3eebc..63e8aaba 100644 --- a/src/scenarios/server/skills/enumeration.ts +++ b/src/scenarios/server/skills/enumeration.ts @@ -132,6 +132,7 @@ function joinErrs(errs: string[], limit = 5): string { export class SkillsEnumerationScenario implements ClientScenario { name = 'sep-2640-skills-enumeration'; readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + readonly requires = { extensions: [SKILLS_EXTENSION_ID] } as const; description = `SEP-2640 Skills extension: \`skills/list\` enumeration and \`skills/get\` retrieval. **Methods**: \`skills/list\`, \`skills/get\` (both mandatory for a server declaring \`io.modelcontextprotocol/skills\`) diff --git a/src/scenarios/server/skills/manifest.ts b/src/scenarios/server/skills/manifest.ts index 00bf4ab2..69207725 100644 --- a/src/scenarios/server/skills/manifest.ts +++ b/src/scenarios/server/skills/manifest.ts @@ -52,6 +52,7 @@ function isNamespacedMetaKey(key: string): boolean { export class SkillsManifestScenario implements ClientScenario { name = 'sep-2640-skills-manifest'; readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + readonly requires = { extensions: [SKILLS_EXTENSION_ID] } as const; description = `SEP-2640 Skills extension: the \`SKILL.md\` manifest resource. **Resource**: \`skill:///SKILL.md\` (read via \`resources/read\`) diff --git a/src/scenarios/server/sse-multiple-streams.ts b/src/scenarios/server/sse-multiple-streams.ts index 91bd24ca..a805ff2f 100644 --- a/src/scenarios/server/sse-multiple-streams.ts +++ b/src/scenarios/server/sse-multiple-streams.ts @@ -23,6 +23,7 @@ import { terminateSessionBestEffort } from '../../connection/sdk-client'; export class ServerSSEMultipleStreamsScenario implements ClientScenario { name = 'server-sse-multiple-streams'; readonly source = { introducedIn: '2025-11-25' } as const; + readonly requires = { capabilities: ['tools'] } as const; description = 'Test server supports multiple concurrent POST SSE streams (SEP-1699)'; diff --git a/src/scenarios/server/sse-polling.ts b/src/scenarios/server/sse-polling.ts index 280fe791..d626a592 100644 --- a/src/scenarios/server/sse-polling.ts +++ b/src/scenarios/server/sse-polling.ts @@ -77,6 +77,7 @@ export class ServerSSEPollingScenario implements ClientScenario { introducedIn: '2025-11-25', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = 'Test server SSE polling via test_reconnection tool that closes stream mid-call (SEP-1699)'; diff --git a/src/scenarios/server/stateless.ts b/src/scenarios/server/stateless.ts index 618dc0c5..76b58e8d 100644 --- a/src/scenarios/server/stateless.ts +++ b/src/scenarios/server/stateless.ts @@ -747,6 +747,16 @@ export class ServerStatelessScenario implements ClientScenario { details: { response: data401 } }; } + if (ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_missing_capability', + response: data401 + } + }; + } return { error: notTestable( "server does not list the diagnostic tool 'test_missing_capability' in tools/list and the probe call did not exercise it (required for the undeclared-capability rejection)" @@ -785,6 +795,16 @@ export class ServerStatelessScenario implements ClientScenario { }; if (!serverRequiresCapability) { + if (!capabilityToolListed && ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_missing_capability', + httpStatus: res401.status + } + }; + } // Same root cause as the previous check: either the diagnostic // tool is missing (untestable) or the server failed to emit the // -32021 this status requirement attaches to. @@ -915,9 +935,22 @@ export class ServerStatelessScenario implements ClientScenario { // requirement could not be exercised — fail with the cause rather // than reporting a vacuous SUCCESS or an invisible SKIPPED. if (frames.every((f) => f?.error !== undefined)) { + const toolListed = listedToolNames.includes( + 'test_streaming_elicitation' + ); + if (!toolListed && ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_streaming_elicitation', + response: frames[0] + } + }; + } return { error: notTestable( - listedToolNames.includes('test_streaming_elicitation') + toolListed ? `the 'test_streaming_elicitation' call was rejected (code ${frames[0]?.error?.code}), so the response stream could not be exercised` : "server does not list the diagnostic tool 'test_streaming_elicitation' in tools/list, so the response stream could not be exercised" ), @@ -962,9 +995,20 @@ export class ServerStatelessScenario implements ClientScenario { // to log and the requirement could not be exercised — fail with the // cause rather than reporting a vacuous SUCCESS or invisible SKIPPED. if (frames.every((f) => f?.error !== undefined)) { + const toolListed = listedToolNames.includes('test_logging_tool'); + if (!toolListed && ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_logging_tool', + response: frames[0] + } + }; + } return { error: notTestable( - listedToolNames.includes('test_logging_tool') + toolListed ? `the 'test_logging_tool' call was rejected (code ${frames[0]?.error?.code}), so the no-log-without-logLevel requirement could not be exercised` : "server does not list the diagnostic tool 'test_logging_tool' in tools/list, so the no-log-without-logLevel requirement could not be exercised" ), @@ -1220,6 +1264,15 @@ export class ServerStatelessScenario implements ClientScenario { } ); if (trigger?.data?.error?.code === -32601) { + if (ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_trigger_prompt_change' + } + }; + } // SHOULD-level requirement, so the untestable report is a WARNING. return { warning: true, @@ -1290,6 +1343,15 @@ export class ServerStatelessScenario implements ClientScenario { } ); if (trigger?.data?.error?.code === -32601) { + if (ctx.production) { + return { + skipped: true, + details: { + skippedReason: 'requires-fixture', + fixture: 'test_trigger_tool_change' + } + }; + } // SHOULD-level requirement, so the untestable report is a WARNING. return { warning: true, diff --git a/src/scenarios/server/tasks/capability.ts b/src/scenarios/server/tasks/capability.ts index 2a0bcda0..d868342f 100644 --- a/src/scenarios/server/tasks/capability.ts +++ b/src/scenarios/server/tasks/capability.ts @@ -20,6 +20,11 @@ import { TASKS_EXTENSION_ID } from './helpers'; export class TasksCapabilityNegotiationScenario implements ClientScenario { name = 'tasks-capability-negotiation'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 capability negotiation for the tasks extension. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/composition.ts b/src/scenarios/server/tasks/composition.ts index 23daebeb..6dee6179 100644 --- a/src/scenarios/server/tasks/composition.ts +++ b/src/scenarios/server/tasks/composition.ts @@ -30,6 +30,11 @@ const MRTR_INPUT_REQUIRED_RESULT_TYPE = 'input_required'; export class TasksMrtrCompositionScenario implements ClientScenario { name = 'tasks-mrtr-composition'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 MRTR → Tasks composition (SEP-2663 commit 451f5e1). **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/dispatch.ts b/src/scenarios/server/tasks/dispatch.ts index 65382fab..0d9fc5aa 100644 --- a/src/scenarios/server/tasks/dispatch.ts +++ b/src/scenarios/server/tasks/dispatch.ts @@ -51,6 +51,11 @@ const ISOLATION_HINT = export class TasksDispatchScenario implements ClientScenario { name = 'tasks-dispatch-and-envelope'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 dispatch / envelope rules across the tasks surface. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/headers.ts b/src/scenarios/server/tasks/headers.ts index d523994d..11991c65 100644 --- a/src/scenarios/server/tasks/headers.ts +++ b/src/scenarios/server/tasks/headers.ts @@ -34,6 +34,11 @@ const HEADER_MISMATCH_ERROR_CODE = HEADER_MISMATCH; export class TasksRequestHeadersScenario implements ClientScenario { name = 'tasks-request-headers'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2243 Mcp-Method / Mcp-Name request-header validation, tasks surface. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/lifecycle.ts b/src/scenarios/server/tasks/lifecycle.ts index 2214ff16..a53c4ff7 100644 --- a/src/scenarios/server/tasks/lifecycle.ts +++ b/src/scenarios/server/tasks/lifecycle.ts @@ -24,6 +24,11 @@ import { isIso8601 } from './mrtr-helpers'; export class TasksLifecycleScenario implements ClientScenario { name = 'tasks-lifecycle'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 Tasks extension lifecycle on the server. **Server Implementation Requirements (SEP-2663):** diff --git a/src/scenarios/server/tasks/mrtr-input.ts b/src/scenarios/server/tasks/mrtr-input.ts index b5b41c61..82744ea4 100644 --- a/src/scenarios/server/tasks/mrtr-input.ts +++ b/src/scenarios/server/tasks/mrtr-input.ts @@ -20,6 +20,11 @@ import { TASKS_EXTENSION_ID, waitForStatus, waitForTerminal } from './helpers'; export class TasksMRTRInputScenario implements ClientScenario { name = 'tasks-mrtr-input'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2322 MRTR input flow on the tasks surface. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/notifications.ts b/src/scenarios/server/tasks/notifications.ts index 6022a704..18f169b7 100644 --- a/src/scenarios/server/tasks/notifications.ts +++ b/src/scenarios/server/tasks/notifications.ts @@ -24,6 +24,11 @@ import { skipCheck } from './mrtr-helpers'; export class TasksStatusNotificationsScenario implements ClientScenario { name = 'tasks-status-notifications'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 status notifications (optional). **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/request-state.ts b/src/scenarios/server/tasks/request-state.ts index 0e6559ca..233fd99e 100644 --- a/src/scenarios/server/tasks/request-state.ts +++ b/src/scenarios/server/tasks/request-state.ts @@ -30,6 +30,11 @@ import { TASKS_EXTENSION_ID } from './helpers'; export class TasksRequestStateRemovalScenario implements ClientScenario { name = 'tasks-request-state-removal'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Verify the absence of \`requestState\` on the tasks-v2 wire. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tasks/required-task-error.ts b/src/scenarios/server/tasks/required-task-error.ts index 1da07c8f..55741a96 100644 --- a/src/scenarios/server/tasks/required-task-error.ts +++ b/src/scenarios/server/tasks/required-task-error.ts @@ -42,6 +42,11 @@ const REQUIRED_TASK_TOOL = 'failing_job'; export class TasksRequiredTaskErrorScenario implements ClientScenario { name = 'tasks-required-task-error'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Verify the -32021 error path for required-task tools when the client has not negotiated the io.modelcontextprotocol/tasks extension. diff --git a/src/scenarios/server/tasks/wire-fields.ts b/src/scenarios/server/tasks/wire-fields.ts index 73684a19..00d82026 100644 --- a/src/scenarios/server/tasks/wire-fields.ts +++ b/src/scenarios/server/tasks/wire-fields.ts @@ -20,6 +20,11 @@ import { untestableCheck } from '../../untestable'; export class TasksWireFieldsScenario implements ClientScenario { name = 'tasks-wire-fields'; readonly source = { extensionId: 'io.modelcontextprotocol/tasks' } as const; + readonly requires = { + extensions: ['io.modelcontextprotocol/tasks'], + capabilities: ['tools'], + fixture: true + } as const; description = `Test SEP-2663 wire-field renames + TTL semantics. **Server Implementation Requirements:** diff --git a/src/scenarios/server/tools.ts b/src/scenarios/server/tools.ts index fd423f1d..c6dedad1 100644 --- a/src/scenarios/server/tools.ts +++ b/src/scenarios/server/tools.ts @@ -249,6 +249,7 @@ export function buildToolsListDeterministicOrderCheck( export class ToolsListScenario implements ClientScenario { name = 'tools-list'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'] } as const; description = `Test listing available tools. **Server Implementation Requirements:** @@ -371,6 +372,7 @@ export class ToolsListScenario implements ClientScenario { export class ToolsCallSimpleTextScenario implements ClientScenario { name = 'tools-call-simple-text'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test calling a tool that returns simple text. **Server Implementation Requirements:** @@ -456,6 +458,7 @@ Implement tool \`test_simple_text\` with no arguments that returns: export class ToolsCallImageScenario implements ClientScenario { name = 'tools-call-image'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test calling a tool that returns image content. **Server Implementation Requirements:** @@ -544,6 +547,7 @@ Implement tool \`test_image_content\` with no arguments that returns: export class ToolsCallMultipleContentTypesScenario implements ClientScenario { name = 'tools-call-mixed-content'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test tool returning multiple content types. **Server Implementation Requirements:** @@ -648,6 +652,10 @@ export class ToolsCallWithLoggingScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { + capabilities: ['tools', 'logging'], + fixture: true + } as const; description = `Test tool that sends log messages during execution. **Server Implementation Requirements:** @@ -738,6 +746,7 @@ Implement tool \`test_tool_with_logging\` with no arguments. export class ToolsCallErrorScenario implements ClientScenario { name = 'tools-call-error'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test tool error reporting. **Server Implementation Requirements:** @@ -823,6 +832,7 @@ Implement tool \`test_error_handling\` with no arguments. export class ToolsCallWithProgressScenario implements ClientScenario { name = 'tools-call-with-progress'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test tool that reports progress notifications. **Server Implementation Requirements:** @@ -947,6 +957,7 @@ export class ToolsCallSamplingScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test tool that requests LLM sampling from client. **Server Implementation Requirements:** @@ -1079,6 +1090,7 @@ export class ToolsCallElicitationScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test tool that requests user input (elicitation) from client. **Server Implementation Requirements:** @@ -1211,6 +1223,7 @@ Implement tool \`test_elicitation\` with argument: export class ToolsCallAudioScenario implements ClientScenario { name = 'tools-call-audio'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test calling a tool that returns audio content. **Server Implementation Requirements:** @@ -1306,6 +1319,7 @@ Implement tool \`test_audio_content\` with no arguments that returns: export class ToolsCallEmbeddedResourceScenario implements ClientScenario { name = 'tools-call-embedded-resource'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { capabilities: ['tools'], fixture: true } as const; description = `Test calling a tool that returns embedded resource content. **Server Implementation Requirements:** diff --git a/src/scenarios/server/utils.ts b/src/scenarios/server/utils.ts index b2297158..115874a6 100644 --- a/src/scenarios/server/utils.ts +++ b/src/scenarios/server/utils.ts @@ -16,6 +16,7 @@ export class LoggingSetLevelScenario implements ClientScenario { introducedIn: '2025-06-18', removedIn: DRAFT_PROTOCOL_VERSION } as const; + readonly requires = { capabilities: ['logging'] } as const; description = `Test setting logging level. **Server Implementation Requirements:** @@ -190,6 +191,10 @@ export class PingScenario implements ClientScenario { export class CompletionCompleteScenario implements ClientScenario { name = 'completion-complete'; readonly source = { introducedIn: '2025-06-18' } as const; + readonly requires = { + capabilities: ['completions'], + fixture: true + } as const; description = `Test completion endpoint. **Server Implementation Requirements:** diff --git a/src/schemas.ts b/src/schemas.ts index b8f67831..2a260fc7 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -37,7 +37,8 @@ export const ServerOptionsSchema = z.object({ .refine((scenario) => getClientScenario(scenario) !== undefined, { error: (iss) => `Unknown scenario '${iss.input}'` }) - .optional() + .optional(), + production: z.boolean().optional() }); export type ServerOptions = z.infer; diff --git a/src/types.ts b/src/types.ts index ebe75a27..69632a14 100644 --- a/src/types.ts +++ b/src/types.ts @@ -119,6 +119,42 @@ export type ScenarioSource = } | { extensionId: ExtensionId }; +/** + * Optional server capabilities declared on `initialize` (2025-x) or + * `server/discover` (2026-x). + */ +export type ServerCapability = + | 'tools' + | 'resources' + | 'resources.subscribe' + | 'prompts' + | 'logging' + | 'completions'; + +/** + * Declarative prerequisites for a server-conformance scenario. + * + * Distinguishes optional MCP capabilities/extensions from reference-fixture + * contracts (`test_*` tools/prompts, `test://` resources). In default SDK + * conformance mode every scenario still runs and missing prerequisites fail; + * in opt-in `--production` mode, unmet capability/extension requirements and + * fixture-bound scenarios emit `SKIPPED` checks instead of `FAILURE`. + */ +export interface ScenarioRequirements { + /** Server capabilities that must all be declared (AND). */ + capabilities?: readonly ServerCapability[]; + /** Server capabilities where at least one must be declared (OR). */ + anyCapability?: readonly ServerCapability[]; + /** Protocol extensions under `capabilities.extensions` that must be declared. */ + extensions?: readonly ExtensionId[]; + /** + * True when the scenario requires reference-fixture artifacts + * (e.g. `test_*` tools/prompts or `test://` resources from the + * conformance everything-server) rather than arbitrary domain endpoints. + */ + fixture?: boolean; +} + export interface ScenarioUrls { serverUrl: string; authUrl?: string; @@ -147,6 +183,7 @@ export interface ClientScenario { name: string; description: string; source: ScenarioSource; + requires?: ScenarioRequirements; run(ctx: RunContext): Promise; }