Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ npx @modelcontextprotocol/conformance server --url <url> [--scenario <scenario>]
- `--scenario <scenario>` - Test scenario to run (e.g., "server-initialize"). Runs all available scenarios by default
- `--suite <suite>` - Suite to run: "active" (default; excludes pending and draft-spec scenarios), "all", "draft" (scenarios targeting the in-progress draft spec), or "pending"
- `--requirements <revision>` - Run exactly what a spec revision requires, frozen at its release (see [Conformance Requirements](#conformance-requirements))
- `--production` - Test a capability-partial production server rather than an SDK reference fixture: scenarios requiring optional capabilities or extensions the server did not declare, or reference-fixture artifacts (`test_*` / `test://`), are reported as `SKIPPED` instead of `FAILURE`
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
- `--verbose` - Show verbose output

Expand Down
5 changes: 5 additions & 0 deletions src/connection/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ export interface Connection {
export interface RunContext {
serverUrl: string;
specVersion: SpecVersion;
/**
* True when running in opt-in `--production` mode against a real deployment
* rather than an SDK reference fixture server.
*/
production?: boolean;
/**
* Open a version-appropriate connection to the server-under-test.
* Scenarios that test the connection mechanics themselves (initialize,
Expand Down
14 changes: 12 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,10 @@ program
'--requirements <revision>',
'Run exactly the scenarios a spec revision requires, frozen at its release (e.g. 2026-07-28). Replaces --suite and --spec-version'
)
.option(
'--production',
'Test a production server: skip scenarios requiring undeclared optional capabilities or reference fixtures'
)
.option('--timeout <ms>', 'Per-scenario timeout in milliseconds', '30000')
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options, cmd) => {
Expand All @@ -564,6 +568,7 @@ program
const timeout = parseInt(options.timeout, 10);

const verbose = options.verbose ?? false;
const production = options.production ?? false;
const outputDir = options.outputDir;
const requirements = resolveRequirements(options.requirements, {
specVersion: options.specVersion,
Expand Down Expand Up @@ -591,7 +596,8 @@ program
outputDir,
specVersionFilter,
options.force ?? false,
timeout
timeout,
{ production }
);

// Inapplicable scenario/spec-version combination (already logged by
Expand Down Expand Up @@ -664,6 +670,9 @@ program

const allResults: { scenario: string; checks: ConformanceCheck[] }[] =
[];
const capabilitiesCache = production
? new Map<SpecVersion, Record<string, unknown>>()
: undefined;

for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
Expand All @@ -679,7 +688,8 @@ program
// a requirement set decides membership, so its choice outranks a
// scenario's own applicability window at the pinned revision
Boolean(requirements),
timeout
timeout,
{ production, capabilitiesCache }
)
);
allResults.push({ scenario: scenarioName, checks: result.checks });
Expand Down
3 changes: 2 additions & 1 deletion src/runner/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ export {
export {
runServerConformanceTest,
printServerResults,
printServerSummary
printServerSummary,
type ServerRunOptions
} from './server';

// Export utilities
Expand Down
Loading