diff --git a/src/scenarios/client/skills/capabilities.test.ts b/src/scenarios/client/skills/capabilities.test.ts index e1bdb90a..639a262b 100644 --- a/src/scenarios/client/skills/capabilities.test.ts +++ b/src/scenarios/client/skills/capabilities.test.ts @@ -1,9 +1,15 @@ import { describe, test, expect } from 'vitest'; import { testScenarioContext } from '../../../mock-server/testing'; import { sendStatelessRequest } from '../../../connection/stateless'; -import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { DRAFT_PROTOCOL_VERSION, type Scenario } from '../../../types'; import { SkillsNoPrefetchScenario } from './no-prefetch'; import { SkillsVerificationScenario } from './verification'; +import { + SkillsLoadByUriScenario, + SkillsInvalidEntryScenario, + SkillsDirectoryUnlistedChildScenario, + SkillsTwoOriginsScenario +} from './hosting'; /** * Regression for PR #330 review: both skills client scenarios declared the @@ -40,13 +46,18 @@ function expectSkillsAdvertised(caps: Caps, where: string): void { expect(caps.resources, `${where}: resources not advertised`).toBeDefined(); } -const SCENARIOS: Array< - [string, () => SkillsNoPrefetchScenario | SkillsVerificationScenario] -> = [ +const SCENARIOS: Array<[string, () => Scenario]> = [ ['no-prefetch', () => new SkillsNoPrefetchScenario()], ['verify-digest', () => new SkillsVerificationScenario('digest')], ['verify-size', () => new SkillsVerificationScenario('size')], - ['verify-frontmatter', () => new SkillsVerificationScenario('frontmatter')] + ['verify-frontmatter', () => new SkillsVerificationScenario('frontmatter')], + ['load-by-uri', () => new SkillsLoadByUriScenario()], + ['invalid-entry', () => new SkillsInvalidEntryScenario()], + [ + 'directory-unlisted-child', + () => new SkillsDirectoryUnlistedChildScenario() + ], + ['two-origins', () => new SkillsTwoOriginsScenario()] ]; describe('SEP-2640 client scenarios advertise skills on both lifecycles', () => { diff --git a/src/scenarios/client/skills/hosting.test.ts b/src/scenarios/client/skills/hosting.test.ts new file mode 100644 index 00000000..ad14fca5 --- /dev/null +++ b/src/scenarios/client/skills/hosting.test.ts @@ -0,0 +1,251 @@ +import { describe, test, expect } from 'vitest'; +import { testScenarioContext } from '../../../mock-server/testing'; +import { sendStatelessRequest } from '../../../connection/stateless'; +import { DRAFT_PROTOCOL_VERSION, type Scenario } from '../../../types'; +import { + SkillsLoadByUriScenario, + SkillsInvalidEntryScenario, + SkillsDirectoryUnlistedChildScenario, + SkillsTwoOriginsScenario +} from './hosting'; + +/** + * Each scenario runs against small scripted clients. A client that follows + * the contract and the spec passes. Each negative client breaks one rule, and + * the matching check must fail. A client that skips the prerequisite gets an + * untestable FAILURE rather than a pass. + */ + +type Call = (method: string, params?: Record) => Promise; + +function caller(url: string): Call { + return async (method, params) => { + const res = await sendStatelessRequest(url, method, params, { + specVersion: DRAFT_PROTOCOL_VERSION, + skipValidation: true + }); + return res.body; + }; +} + +async function run( + scenario: Scenario, + client: (call: Call, context: any, second?: Call) => Promise +) { + const urls = await scenario.start( + testScenarioContext(DRAFT_PROTOCOL_VERSION) + ); + try { + const second = urls.context?.secondServerUrl as string | undefined; + await client( + caller(urls.serverUrl), + urls.context, + second ? caller(second) : undefined + ); + return new Map(scenario.getChecks().map((c) => [c.id, c])); + } finally { + await scenario.stop(); + } +} + +/** Reads every file an entry lists, which is what a verifying load does. */ +async function load(call: Call, entry: any) { + for (const r of entry.resources ?? []) { + await call('resources/read', { uri: r.uri }); + } +} + +const listed = async (call: Call) => + ((await call('skills/list')).result?.skills ?? []) as any[]; + +describe('sep-2640-client-load-by-uri', () => { + const ids = [ + 'sep-2640-host-load-by-uri', + 'sep-2640-host-no-empty-listing-assumption' + ]; + + test('listing, then skills/get and a read, passes both checks', async () => { + const checks = await run( + new SkillsLoadByUriScenario(), + async (call, ctx) => { + await listed(call); + const got = await call('skills/get', { uri: ctx.skillUri }); + await call('resources/read', { uri: got.result.skill.uri }); + } + ); + expect(ids.map((id) => checks.get(id)?.status)).toEqual([ + 'SUCCESS', + 'SUCCESS' + ]); + }); + + test('stopping at the empty listing fails both checks', async () => { + const checks = await run(new SkillsLoadByUriScenario(), async (call) => { + await listed(call); + }); + expect(ids.map((id) => checks.get(id)?.status)).toEqual([ + 'FAILURE', + 'FAILURE' + ]); + }); + + test('resolving the entry without reading the SKILL.md fails load-by-uri', async () => { + const checks = await run( + new SkillsLoadByUriScenario(), + async (call, ctx) => { + await listed(call); + await call('skills/get', { uri: ctx.skillUri }); + } + ); + expect(checks.get('sep-2640-host-load-by-uri')?.status).toBe('FAILURE'); + }); +}); + +describe('sep-2640-client-invalid-entry', () => { + const id = 'sep-2640-resources-invalid-entry'; + const isValid = (e: any) => + Array.isArray(e.resources) || e.resources === 'dynamic'; + + test('loading only the valid entry passes', async () => { + const checks = await run(new SkillsInvalidEntryScenario(), async (call) => { + for (const e of await listed(call)) if (isValid(e)) await load(call, e); + }); + expect(checks.get(id)?.status).toBe('SUCCESS'); + }); + + test('loading an entry with no resources fails', async () => { + const checks = await run(new SkillsInvalidEntryScenario(), async (call) => { + for (const e of await listed(call)) { + await call('resources/read', { uri: e.uri }); + } + }); + expect(checks.get(id)?.status).toBe('FAILURE'); + }); + + test('loading nothing is untestable, not a pass', async () => { + const checks = await run(new SkillsInvalidEntryScenario(), async (call) => { + await listed(call); + }); + expect(checks.get(id)?.status).toBe('FAILURE'); + expect(checks.get(id)?.details?.untestable).toBe(true); + }); +}); + +describe('sep-2640-client-directory-unlisted-child', () => { + const id = 'sep-2640-directory-read-not-manifest-extension'; + + async function walk( + call: Call, + readChild: (uri: string, entry: any) => boolean + ) { + const [entry] = await listed(call); + await load(call, entry); + const queue = ['skill://pdf-processing']; + while (queue.length) { + const dir = await call('resources/directory/read', { + uri: queue.shift() + }); + for (const c of dir.result?.resources ?? []) { + if (c.mimeType === 'inode/directory') queue.push(c.uri); + else if (readChild(c.uri, entry)) + await call('resources/read', { uri: c.uri }); + } + } + } + + test('reading only children the entry lists passes', async () => { + const checks = await run( + new SkillsDirectoryUnlistedChildScenario(), + async (call) => { + await walk(call, (uri, entry) => + entry.resources.some((r: any) => r.uri === uri) + ); + } + ); + expect(checks.get(id)?.status).toBe('SUCCESS'); + }); + + test('reading a child the entry does not list fails', async () => { + const checks = await run( + new SkillsDirectoryUnlistedChildScenario(), + async (call) => { + await walk(call, () => true); + } + ); + expect(checks.get(id)?.status).toBe('FAILURE'); + }); +}); + +describe('sep-2640-client-two-origins', () => { + const ids = [ + 'sep-2640-host-skill-identity-pair', + 'sep-2640-host-cross-origin-no-shadow', + 'sep-2640-host-reads-bound-to-origin' + ]; + + test('loading each server’s skills from that server passes every check', async () => { + const checks = await run( + new SkillsTwoOriginsScenario(), + async (a, _c, b) => { + for (const call of [a, b!]) { + for (const e of await listed(call)) await load(call, e); + } + } + ); + expect(ids.map((id) => checks.get(id)?.status)).toEqual([ + 'SUCCESS', + 'SUCCESS', + 'SUCCESS' + ]); + }); + + test('caching by uri alone fails identity-pair', async () => { + const checks = await run( + new SkillsTwoOriginsScenario(), + async (a, _c, b) => { + const seen = new Set(); + for (const call of [a, b!]) { + for (const e of await listed(call)) { + if (seen.has(e.uri)) continue; + seen.add(e.uri); + await load(call, e); + } + } + } + ); + expect(checks.get(ids[0])?.status).toBe('FAILURE'); + }); + + test('keying skills by name alone fails cross-origin-no-shadow', async () => { + const checks = await run( + new SkillsTwoOriginsScenario(), + async (a, _c, b) => { + const byName = new Map(); + for (const call of [a, b!]) { + for (const e of await listed(call)) + byName.set(e.frontmatter.name, [call, e]); + } + for (const [call, e] of byName.values()) await load(call, e); + } + ); + expect(checks.get(ids[1])?.status).toBe('FAILURE'); + }); + + test('reading one server’s files from the other fails reads-bound-to-origin', async () => { + const checks = await run( + new SkillsTwoOriginsScenario(), + async (a, _c, b) => { + const all: any[] = [...(await listed(a)), ...(await listed(b!))]; + for (const call of [a, b!]) for (const e of all) await load(call, e); + } + ); + expect(checks.get(ids[2])?.status).toBe('FAILURE'); + }); + + test('listing only one server is untestable', async () => { + const checks = await run(new SkillsTwoOriginsScenario(), async (a) => { + for (const e of await listed(a)) await load(a, e); + }); + expect(ids.every((id) => checks.get(id)?.details?.untestable)).toBe(true); + }); +}); diff --git a/src/scenarios/client/skills/hosting.ts b/src/scenarios/client/skills/hosting.ts new file mode 100644 index 00000000..cb9ab844 --- /dev/null +++ b/src/scenarios/client/skills/hosting.ts @@ -0,0 +1,433 @@ +/** + * SEP-2640 client scenarios for what a host loads, refuses, and reads from + * which server. The harness is the server, and every scenario grades the + * requests the client sends. + * + * Each contract tells the client to try something, and the client library is + * what must refuse it. A client that loads an invalid entry, or reads a file a + * directory listed but the entry did not, sends a request the scenario sees. + * That separates these from the retired `unlisted` variant in verification.ts: + * there the driver was told not to ask, so only the driver was under test. + * + * Absence of a request proves something only once the client has done the + * work around it, so every scenario requires a prerequisite request and + * reports untestable without it (#248). + */ + +import { ConformanceCheck, ScenarioUrls, Scenario } from '../../../types.js'; +import type { ScenarioContext } from '../../../mock-server'; +import { untestableCheck } from '../../untestable.js'; +import { + SKILLS_EXTENSION_ID, + SkillsMockHost, + entryFor, + skillMd +} from './mock-host.js'; + +const SPEC_URL = + 'https://skills.extensions.modelcontextprotocol.io/specification/beta/skills'; + +function ref(anchor: string) { + return { id: `SEP-2640-${anchor}`, url: `${SPEC_URL}#${anchor}` }; +} + +function check( + id: string, + description: string, + anchor: string, + violation: string | undefined, + details: Record +): ConformanceCheck { + return { + id, + name: id, + description, + status: violation ? 'FAILURE' : 'SUCCESS', + timestamp: new Date().toISOString(), + errorMessage: violation, + specReferences: [ref(anchor)], + details + }; +} + +function untestable( + id: string, + description: string, + anchor: string, + reason: string +): ConformanceCheck { + return untestableCheck(id, id, description, reason, [ref(anchor)]); +} + +/** A scenario backed by one SkillsMockHost, plus optional context. */ +abstract class OneHostScenario implements Scenario { + abstract name: string; + abstract description: string; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + allowClientError = true; + protected abstract host: SkillsMockHost; + protected context(): Record | undefined { + return undefined; + } + + async start(ctx: ScenarioContext): Promise { + const urls = await this.host.start(ctx); + const context = this.context(); + return context ? { ...urls, context } : urls; + } + + async stop(): Promise { + await this.host.stop(); + } + + abstract getChecks(): ConformanceCheck[]; +} + +// --- load by URI after an empty listing ------------------------------------- + +const BY_URI = 'skill://pdf-processing/SKILL.md'; +const BY_URI_FILES = { [BY_URI]: skillMd('pdf-processing') }; + +/** + * `skills/list` answers an empty listing, and `skills/get` resolves one skill + * the scenario names in `MCP_CONFORMANCE_CONTEXT` as `skillUri`. + * + * Contract: call `skills/list`, then load the skill at `skillUri` by its URI + * (`skills/get`, then read its SKILL.md), then exit. + */ +export class SkillsLoadByUriScenario extends OneHostScenario { + name = 'sep-2640-client-load-by-uri'; + description = + 'A host MUST support loading a skill given only its URI, and MUST NOT treat an empty listing as proof that a server has no skills.'; + protected host = new SkillsMockHost(this.name, { + listed: [], + gettable: [entryFor(BY_URI, 'pdf-processing', BY_URI_FILES)], + files: BY_URI_FILES + }); + + protected context() { + return { skillUri: BY_URI }; + } + + getChecks(): ConformanceCheck[] { + const listIdx = this.host.requests.findIndex( + (r) => r.method === 'skills/list' + ); + const loadIdx = this.host.requests.findIndex( + (r) => + (r.method === 'skills/get' || r.method === 'resources/read') && + r.uri === BY_URI + ); + const got = this.host.called('skills/get', BY_URI); + const read = this.host.called('resources/read', BY_URI); + const details = { requests: this.host.requests }; + + const emptyText = + 'Hosts MUST NOT treat an empty or partial listing as proof that a server has no skills.'; + const empty = + listIdx < 0 + ? untestable( + 'sep-2640-host-no-empty-listing-assumption', + emptyText, + 'listing-skills', + 'the client never called skills/list, so it never saw the empty listing' + ) + : check( + 'sep-2640-host-no-empty-listing-assumption', + emptyText, + 'listing-skills', + loadIdx > listIdx + ? undefined + : `the client got an empty skills/list and never asked for ${BY_URI}, which the server serves`, + details + ); + + const byUriText = + 'Hosts MUST support loading a skill given only its URI: skills/get resolves the entry, and the SKILL.md is read against it.'; + const byUri = check( + 'sep-2640-host-load-by-uri', + byUriText, + 'listing-skills', + got && read + ? undefined + : !got + ? `the client never called skills/get for ${BY_URI}` + : `the client resolved ${BY_URI} with skills/get but never read the SKILL.md`, + details + ); + return [empty, byUri]; + } +} + +// --- invalid entries are not loaded ----------------------------------------- + +const VALID = 'skill://pdf-processing/SKILL.md'; +const NO_RESOURCES = 'skill://no-manifest/SKILL.md'; +const BAD_RESOURCES = 'skill://bad-manifest/SKILL.md'; + +/** + * `skills/list` answers one valid entry and two invalid ones: one with no + * `resources`, and one whose `resources` is neither an array nor "dynamic". + * The server still serves all three SKILL.md files. + * + * Contract: call `skills/list`, then load every listed skill, then exit. + */ +export class SkillsInvalidEntryScenario extends OneHostScenario { + name = 'sep-2640-client-invalid-entry'; + description = + 'An entry with no resources, or with resources neither an array nor "dynamic", is invalid, and hosts MUST NOT load it.'; + protected host = (() => { + const files = { + [VALID]: skillMd('pdf-processing'), + [NO_RESOURCES]: skillMd('no-manifest'), + [BAD_RESOURCES]: skillMd('bad-manifest') + }; + const noResources = entryFor(NO_RESOURCES, 'no-manifest', {}); + delete noResources.resources; + return new SkillsMockHost(this.name, { + listed: [ + entryFor(VALID, 'pdf-processing', { [VALID]: files[VALID] }), + noResources, + { ...entryFor(BAD_RESOURCES, 'bad-manifest', {}), resources: 'all' } + ], + files + }); + })(); + + getChecks(): ConformanceCheck[] { + const id = 'sep-2640-resources-invalid-entry'; + const text = this.description; + if (!this.host.called('resources/read', VALID)) { + return [ + untestable( + id, + text, + 'resources', + `the client never loaded ${VALID}, the valid entry, so not loading the invalid ones proves nothing` + ) + ]; + } + const loaded = [NO_RESOURCES, BAD_RESOURCES].filter((u) => + this.host.called('resources/read', u) + ); + return [ + check( + id, + text, + 'resources', + loaded.length + ? `the client loaded ${loaded.join(' and ')}, whose entries are invalid` + : undefined, + { reads: this.host.reads() } + ) + ]; + } +} + +// --- a directory listing does not extend the manifest ----------------------- + +const DIR_SKILL = 'skill://pdf-processing/SKILL.md'; +const DIR_LISTED = 'skill://pdf-processing/references/FORMS.md'; +const DIR_EXTRA = 'skill://pdf-processing/references/EXTRA.md'; + +/** + * The entry lists SKILL.md and references/FORMS.md. `resources/directory/read` + * of references/ lists FORMS.md and EXTRA.md, and the server serves both. + * + * Contract: call `skills/list`, load the skill, walk its directory with + * `resources/directory/read` from the skill root, and read every file the + * directory lists through the held entry. Then exit. + */ +export class SkillsDirectoryUnlistedChildScenario extends OneHostScenario { + name = 'sep-2640-client-directory-unlisted-child'; + description = + 'Hosts MUST NOT treat a directory result as extending the manifest, and MUST NOT read a child the entry does not list.'; + protected host = (() => { + const files = { + [DIR_SKILL]: skillMd('pdf-processing'), + [DIR_LISTED]: 'Listed supporting file.\n', + [DIR_EXTRA]: 'A file the entry does not list.\n' + }; + return new SkillsMockHost(this.name, { + listed: [ + entryFor(DIR_SKILL, 'pdf-processing', { + [DIR_SKILL]: files[DIR_SKILL], + [DIR_LISTED]: files[DIR_LISTED] + }) + ], + files, + dirs: { + 'skill://pdf-processing': [ + { uri: DIR_SKILL, name: 'SKILL.md', mimeType: 'text/markdown' }, + { + uri: 'skill://pdf-processing/references', + name: 'references', + mimeType: 'inode/directory' + } + ], + 'skill://pdf-processing/references': [ + { uri: DIR_LISTED, name: 'FORMS.md', mimeType: 'text/markdown' }, + { uri: DIR_EXTRA, name: 'EXTRA.md', mimeType: 'text/markdown' } + ] + } + }); + })(); + + getChecks(): ConformanceCheck[] { + const id = 'sep-2640-directory-read-not-manifest-extension'; + const anchor = 'directory-reads-and-the-held-entry'; + if ( + !this.host.called( + 'resources/directory/read', + 'skill://pdf-processing/references' + ) || + !this.host.called('resources/read', DIR_LISTED) + ) { + return [ + untestable( + id, + this.description, + anchor, + `the client did not read references/ and then ${DIR_LISTED}, so it never reached the unlisted child` + ) + ]; + } + return [ + check( + id, + this.description, + anchor, + this.host.called('resources/read', DIR_EXTRA) + ? `the client read ${DIR_EXTRA}, which the directory listed and the entry did not` + : undefined, + { reads: this.host.reads() } + ) + ]; + } +} + +// --- two servers ------------------------------------------------------------ + +const SHARED = 'skill://shared/SKILL.md'; +const A_ONLY_FILE = 'skill://shared/references/A.md'; +const B_ONLY_FILE = 'skill://shared/references/B.md'; +const A_PDF = 'skill://pdf/SKILL.md'; +const B_PDF = 'skill://acme/pdf/SKILL.md'; + +function twoOriginHost(name: string, side: 'A' | 'B'): SkillsMockHost { + const shared = skillMd('shared', `Served by server ${side}.`); + const own = side === 'A' ? A_ONLY_FILE : B_ONLY_FILE; + const pdfUri = side === 'A' ? A_PDF : B_PDF; + const pdf = skillMd('pdf', `The pdf skill on server ${side}.`); + const sharedFiles = { [SHARED]: shared, [own]: `Only on ${side}.\n` }; + return new SkillsMockHost(`${name}-${side}`, { + listed: [ + entryFor(SHARED, 'shared', sharedFiles), + entryFor(pdfUri, 'pdf', { [pdfUri]: pdf }) + ], + files: { ...sharedFiles, [pdfUri]: pdf } + }); +} + +/** + * Two servers. Both serve skill://shared/SKILL.md, with different bodies and + * a supporting file only that server has. Each also serves a skill named + * `pdf`, at skill://pdf/SKILL.md on A and skill://acme/pdf/SKILL.md on B. + * Server A is `serverUrl`, and server B arrives in `MCP_CONFORMANCE_CONTEXT` + * as `secondServerUrl`. + * + * Contract: connect to both, call `skills/list` on each, and load every + * listed skill from the server that listed it, reading every file its entry + * lists. Then exit. + */ +export class SkillsTwoOriginsScenario implements Scenario { + name = 'sep-2640-client-two-origins'; + description = + "A skill's identity is the pair of its server and its uri. Reads stay on the originating server, and a same-named skill on another server is not shadowed."; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + allowClientError = true; + private a = twoOriginHost(this.name, 'A'); + private b = twoOriginHost(this.name, 'B'); + + async start(ctx: ScenarioContext): Promise { + const a = await this.a.start(ctx); + const b = await this.b.start(ctx); + return { + serverUrl: a.serverUrl, + context: { secondServerUrl: b.serverUrl } + }; + } + + async stop(): Promise { + await Promise.all([this.a.stop(), this.b.stop()]); + } + + getChecks(): ConformanceCheck[] { + const ids: Array<[string, string, string]> = [ + [ + 'sep-2640-host-skill-identity-pair', + 'skill-uris', + 'The identity of an MCP-served skill is the pair of the server and the uri; hosts MUST NOT key it on the uri alone.' + ], + [ + 'sep-2640-host-cross-origin-no-shadow', + 'names', + 'An MCP-served skill MUST NOT silently shadow, or be substituted for, a same-named skill from another origin.' + ], + [ + 'sep-2640-host-reads-bound-to-origin', + 'security-considerations', + "Hosts MUST bind reads to the skill's originating server. A skill served by A MUST NOT cause a resources/read against B." + ] + ]; + if (!this.a.called('skills/list') || !this.b.called('skills/list')) { + return ids.map(([id, anchor, text]) => + untestable( + id, + text, + anchor, + 'the client did not call skills/list on both servers, so it never held skills from two origins' + ) + ); + } + const details = { readsA: this.a.reads(), readsB: this.b.reads() }; + const identity = + this.a.called('resources/read', SHARED) && + this.b.called('resources/read', SHARED) + ? undefined + : `${SHARED} was read from ${this.a.called('resources/read', SHARED) ? 'server A only' : this.b.called('resources/read', SHARED) ? 'server B only' : 'neither server'}; both serve their own skill at that uri`; + const shadow = + this.a.called('resources/read', A_PDF) && + this.b.called('resources/read', B_PDF) + ? undefined + : `both servers list a skill named pdf, and the client loaded ${ + [ + this.a.called('resources/read', A_PDF) && 'A’s', + this.b.called('resources/read', B_PDF) && 'B’s' + ] + .filter(Boolean) + .join(' and ') || 'neither' + }`; + const strays = [ + ...this.a + .reads() + .filter((u) => u === B_ONLY_FILE || u === B_PDF) + .map((u) => `server A got a read of ${u}, which only B lists`), + ...this.b + .reads() + .filter((u) => u === A_ONLY_FILE || u === A_PDF) + .map((u) => `server B got a read of ${u}, which only A lists`) + ]; + return [ + check(ids[0][0], ids[0][2], ids[0][1], identity, details), + check(ids[1][0], ids[1][2], ids[1][1], shadow, details), + check( + ids[2][0], + ids[2][2], + ids[2][1], + strays.length ? strays.join('; ') : undefined, + details + ) + ]; + } +} diff --git a/src/scenarios/client/skills/mock-host.ts b/src/scenarios/client/skills/mock-host.ts new file mode 100644 index 00000000..71f590b1 --- /dev/null +++ b/src/scenarios/client/skills/mock-host.ts @@ -0,0 +1,177 @@ +/** + * A configurable SEP-2640 server for the client scenarios that grade what a + * host does with a listing: what it loads, what it refuses, and which server + * it reads from. It serves a fixed catalogue and records every request, and + * the scenarios grade that log. + * + * One scenario can run two of these. `start()` returns one `serverUrl` per + * instance, and a scenario hands the second one to the client in + * `ScenarioUrls.context`, the way the auth scenarios hand over their + * authorization server. + */ + +import { createHash } from 'crypto'; +import http from 'http'; +import type { ConformanceCheck } from '../../../types.js'; +import { BaseHttpScenario } from '../http-base.js'; +import { + initializeResult, + readResult, + skillsListResult +} from './mock-results.js'; + +export const SKILLS_EXTENSION_ID = 'io.modelcontextprotocol/skills'; + +const JSONRPC_INVALID_PARAMS = -32602; + +export const sha256 = (s: string) => + 'sha256:' + createHash('sha256').update(s, 'utf8').digest('hex'); + +/** A SKILL.md with the frontmatter an entry for it must carry. */ +export function skillMd(name: string, body = 'Skill body.'): string { + return `---\nname: ${name}\ndescription: The ${name} fixture skill\n---\n\n${body}\n`; +} + +/** + * An honest entry: `resources` lists every file in `files` with its real + * digest and size, and `frontmatter` matches the SKILL.md at `uri`. + */ +export function entryFor( + uri: string, + name: string, + files: Record +): Record { + return { + uri, + frontmatter: { name, description: `The ${name} fixture skill` }, + resources: Object.entries(files).map(([u, text]) => ({ + uri: u, + digest: sha256(text), + size: Buffer.byteLength(text) + })) + }; +} + +export interface MockHostConfig { + /** What `skills/list` answers. */ + listed: object[]; + /** What `skills/get` resolves, by `uri`. Defaults to `listed`. */ + gettable?: object[]; + /** Every file `resources/read` serves, by URI. */ + files: Record; + /** Directory URI to its children. Declares `directoryRead` when present. */ + dirs?: Record; +} + +export interface LoggedRequest { + method: string; + uri?: string; +} + +export class SkillsMockHost extends BaseHttpScenario { + name: string; + description = 'A SEP-2640 server that records what the client asks for.'; + readonly source = { extensionId: SKILLS_EXTENSION_ID } as const; + readonly requests: LoggedRequest[] = []; + + constructor( + name: string, + private readonly config: MockHostConfig + ) { + super(); + this.name = name; + } + + /** Every `resources/read` URI, in order. */ + reads(): string[] { + return this.requests + .filter((r) => r.method === 'resources/read' && r.uri !== undefined) + .map((r) => r.uri!); + } + + called(method: string, uri?: string): boolean { + return this.requests.some( + (r) => r.method === method && (uri === undefined || r.uri === uri) + ); + } + + getChecks(): ConformanceCheck[] { + return []; + } + + protected discoverCapabilities(): object { + const settings = this.config.dirs ? { directoryRead: true } : {}; + return { + resources: { listChanged: false }, + extensions: { [SKILLS_EXTENSION_ID]: settings } + }; + } + + protected handlePost( + _req: http.IncomingMessage, + res: http.ServerResponse, + request: any + ): void { + const method = request.method as string; + const uri = + typeof request.params?.uri === 'string' ? request.params.uri : undefined; + if (request.id !== undefined) this.requests.push({ method, uri }); + const reply = (result: unknown) => + this.sendJson(res, { jsonrpc: '2.0', id: request.id, result }); + const invalid = (message: string) => + this.sendJson(res, { + jsonrpc: '2.0', + id: request.id, + error: { code: JSONRPC_INVALID_PARAMS, message } + }); + + switch (method) { + case 'initialize': + reply( + initializeResult(this.name, request, this.discoverCapabilities()) + ); + return; + case 'skills/list': + reply(skillsListResult(this.config.listed)); + return; + case 'skills/get': { + const pool = (this.config.gettable ?? this.config.listed) as Array<{ + uri?: unknown; + }>; + const skill = pool.find((e) => e.uri === uri); + if (!skill) return invalid(`no skill at ${uri}`); + reply({ + resultType: 'complete', + ttlMs: 0, + cacheScope: 'private', + skill + }); + return; + } + case 'resources/read': { + const text = uri !== undefined ? this.config.files[uri] : undefined; + if (text === undefined) return invalid(`${uri} is not served here`); + reply(readResult(uri, text)); + return; + } + case 'resources/directory/read': { + const children = + uri !== undefined ? this.config.dirs?.[uri] : undefined; + if (!children) return invalid(`${uri} is not a directory`); + reply({ + resultType: 'complete', + ttlMs: 0, + cacheScope: 'private', + resources: children + }); + return; + } + default: + if (request.id === undefined) { + this.sendNotificationAck(res); + return; + } + this.sendGenericResult(res, request); + } + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index ff377f54..604ed7ef 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -12,6 +12,12 @@ import { import { InitializeScenario } from './client/initialize'; import { SkillsNoPrefetchScenario } from './client/skills/no-prefetch'; import { SkillsVerificationScenario } from './client/skills/verification'; +import { + SkillsLoadByUriScenario, + SkillsInvalidEntryScenario, + SkillsDirectoryUnlistedChildScenario, + SkillsTwoOriginsScenario +} from './client/skills/hosting'; import { ToolsCallScenario } from './client/tools_call'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; import { SSERetryScenario } from './client/sse-retry'; @@ -347,7 +353,11 @@ const scenariosList: Scenario[] = [ new SkillsNoPrefetchScenario(), new SkillsVerificationScenario('digest'), new SkillsVerificationScenario('size'), - new SkillsVerificationScenario('frontmatter') + new SkillsVerificationScenario('frontmatter'), + new SkillsLoadByUriScenario(), + new SkillsInvalidEntryScenario(), + new SkillsDirectoryUnlistedChildScenario(), + new SkillsTwoOriginsScenario() ]; // Core scenarios (tier 1 requirements) diff --git a/src/scenarios/server/skills/coverage.test.ts b/src/scenarios/server/skills/coverage.test.ts new file mode 100644 index 00000000..5a9b9ea7 --- /dev/null +++ b/src/scenarios/server/skills/coverage.test.ts @@ -0,0 +1,283 @@ +import { describe, test, expect } from 'vitest'; +import { createServer, type IncomingMessage, type Server } from 'http'; +import type { AddressInfo } from 'net'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { SkillsEnumerationScenario } from './enumeration'; +import { SkillsDirectoryReadScenario } from './directory'; +import { SKILLS_EXTENSION_ID } from './helpers'; + +/** + * Positive and negative controls for the nested-skill, error-handling and + * directory checks. The fixture gets every rule right: acme encloses billing + * and lists billing's SKILL.md, an unserved file answers -32602 with a reason, + * and the acme directory holds an empty subdirectory. Each negative test + * breaks one rule and asserts that the matching check fails. + */ + +const DIR = 'inode/directory'; +const ZERO = `sha256:${'0'.repeat(64)}`; +const OUTER = 'skill://acme/SKILL.md'; +const INNER = 'skill://acme/billing/SKILL.md'; + +interface Fixture { + skills: object[]; + /** Directory URI to its children, or 'omit' to answer with no resources member. */ + dirs: Record; + /** The error resources/read answers for a file the server does not serve. */ + unserved: { code: number; message: string } | 'succeed'; + dirCache: boolean; +} + +function goodFixture(): Fixture { + return { + skills: [ + { + uri: OUTER, + frontmatter: { name: 'acme', description: 'The enclosing skill.' }, + resources: [ + { uri: OUTER, digest: ZERO, size: 1 }, + { uri: INNER, digest: ZERO, size: 1 } + ] + }, + { + uri: INNER, + frontmatter: { name: 'billing', description: 'A nested skill.' }, + resources: [{ uri: INNER, digest: ZERO, size: 1 }] + } + ], + dirs: { + 'skill://acme': [ + { uri: OUTER, name: 'SKILL.md', mimeType: 'text/markdown' }, + { uri: 'skill://acme/billing', name: 'billing', mimeType: DIR }, + { uri: 'skill://acme/empty', name: 'empty', mimeType: DIR } + ], + 'skill://acme/billing': [ + { uri: INNER, name: 'SKILL.md', mimeType: 'text/markdown' } + ], + 'skill://acme/empty': [] + }, + unserved: { code: -32602, message: 'skill://acme has no file named that' }, + dirCache: true + }; +} + +async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +function startServer(f: Fixture): Promise<{ url: string; server: Server }> { + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const uri = (body.params as { uri?: string } | undefined)?.uri; + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (code: number, message: string) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ jsonrpc: '2.0', id, error: { code, message } })); + }; + + switch (method) { + case 'server/discover': + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: { + resources: {}, + extensions: { [SKILLS_EXTENSION_ID]: { directoryRead: true } } + }, + serverInfo: { name: 'skills-coverage', version: '1.0.0' } + }); + return; + case 'skills/list': + send({ skills: f.skills, ttlMs: 0, cacheScope: 'private' }); + return; + case 'skills/get': { + const skill = (f.skills as Array<{ uri: string }>).find( + (s) => s.uri === uri + ); + if (!skill) return fail(-32602, `no skill at ${uri}`); + send({ skill, ttlMs: 0, cacheScope: 'private' }); + return; + } + case 'resources/list': + send({ resources: [] }); + return; + case 'resources/read': + if (uri === OUTER || uri === INNER) { + const name = uri === OUTER ? 'acme' : 'billing'; + send({ + contents: [ + { + uri, + mimeType: 'text/markdown', + text: `---\nname: ${name}\ndescription: x\n---\n` + } + ] + }); + return; + } + if (f.unserved === 'succeed') { + send({ contents: [{ uri, mimeType: 'text/markdown', text: 'x' }] }); + return; + } + return fail(f.unserved.code, f.unserved.message); + case 'resources/directory/read': { + const children = uri !== undefined ? f.dirs[uri] : undefined; + if (children === undefined) + return fail(-32602, `${uri} is not a directory`); + const cache = f.dirCache ? { ttlMs: 0, cacheScope: 'private' } : {}; + send(children === 'omit' ? cache : { resources: children, ...cache }); + return; + } + } + fail(-32601, `Method not found: ${method}`); + }); + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, () => { + const addr = server.address() as AddressInfo; + resolve({ url: `http://localhost:${addr.port}/mcp`, server }); + }); + }); +} + +async function checksFor(f: Fixture) { + const { url, server } = await startServer(f); + try { + const ctx = () => testContext(url, DRAFT_PROTOCOL_VERSION); + const checks = [ + ...(await new SkillsEnumerationScenario().run(ctx())), + ...(await new SkillsDirectoryReadScenario().run(ctx())) + ]; + // Broken fixtures may break the base schema too; these tests assert the + // scenarios' own checks. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await new Promise((r) => server.close(() => r())); + } +} + +const statusOf = async (f: Fixture, id: string) => + (await checksFor(f)).get(id)?.status; + +const NEW_IDS = [ + 'sep-2640-resources-nested-completeness', + 'sep-2640-nested-publication-flat', + 'sep-2640-error-standard-codes', + 'sep-2640-error-informative-messages', + 'sep-2640-directory-read-cache-attributes', + 'sep-2640-directory-read-empty-dir' +]; + +describe('SEP-2640 nested skills, errors and directory coverage', () => { + test('a server that gets every rule right passes every new check', async () => { + const checks = await checksFor(goodFixture()); + const notPassing = NEW_IDS.map((id) => checks.get(id)) + .filter((c) => c?.status !== 'SUCCESS') + .map((c) => `${c?.id}: ${c?.status} ${c?.errorMessage ?? ''}`); + expect(notPassing).toEqual([]); + }); + + test('an enclosing skill that leaves out the nested SKILL.md fails nested-completeness', async () => { + const f = goodFixture(); + (f.skills[0] as { resources: object[] }).resources = [ + { uri: OUTER, digest: ZERO, size: 1 } + ]; + expect(await statusOf(f, 'sep-2640-resources-nested-completeness')).toBe( + 'FAILURE' + ); + }); + + test('an entry that embeds the nested entry fails nested-publication-flat', async () => { + const f = goodFixture(); + f.skills[0] = { ...f.skills[0], children: [f.skills[1]] }; + expect(await statusOf(f, 'sep-2640-nested-publication-flat')).toBe( + 'FAILURE' + ); + }); + + test('a listing with no nested skill SKIPs both nesting checks', async () => { + const f = goodFixture(); + f.skills = [f.skills[0]]; + const checks = await checksFor(f); + expect(checks.get('sep-2640-resources-nested-completeness')?.status).toBe( + 'SKIPPED' + ); + expect(checks.get('sep-2640-nested-publication-flat')?.status).toBe( + 'SKIPPED' + ); + }); + + test('-32002 for an unserved file fails error-standard-codes on 2026-07-28', async () => { + const f = goodFixture(); + f.unserved = { code: -32002, message: 'no such resource' }; + expect(await statusOf(f, 'sep-2640-error-standard-codes')).toBe('FAILURE'); + }); + + test('serving a file the skill does not list fails error-standard-codes', async () => { + expect( + await statusOf( + { ...goodFixture(), unserved: 'succeed' }, + 'sep-2640-error-standard-codes' + ) + ).toBe('FAILURE'); + }); + + test('an error message that only names the code is a WARNING', async () => { + const f = goodFixture(); + f.unserved = { code: -32602, message: 'Invalid params' }; + expect(await statusOf(f, 'sep-2640-error-informative-messages')).toBe( + 'WARNING' + ); + }); + + test('a directory read without ttlMs and cacheScope fails directory-read-cache-attributes', async () => { + expect( + await statusOf( + { ...goodFixture(), dirCache: false }, + 'sep-2640-directory-read-cache-attributes' + ) + ).toBe('FAILURE'); + }); + + test('an empty directory answered with no resources member fails directory-read-empty-dir', async () => { + const f = goodFixture(); + f.dirs['skill://acme/empty'] = 'omit'; + expect(await statusOf(f, 'sep-2640-directory-read-empty-dir')).toBe( + 'FAILURE' + ); + }); + + test('a server with no empty directory SKIPs directory-read-empty-dir', async () => { + const f = goodFixture(); + f.dirs['skill://acme'] = (f.dirs['skill://acme'] as object[]).slice(0, 2); + expect(await statusOf(f, 'sep-2640-directory-read-empty-dir')).toBe( + 'SKIPPED' + ); + }); +}); diff --git a/src/scenarios/server/skills/directory.ts b/src/scenarios/server/skills/directory.ts index 91dae452..cd8ad059 100644 --- a/src/scenarios/server/skills/directory.ts +++ b/src/scenarios/server/skills/directory.ts @@ -2,7 +2,7 @@ * SEP-2640 Skills extension — the `resources/directory/read` surface (added in * spec commit 2e04c48d, 2026-06-09). * - * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). + * One scenario, eight checks (per AGENTS.md "fewer scenarios, more checks"). * Each check's verbatim spec quote lives next to its check ID in * src/seps/sep-2640.yaml. * @@ -45,6 +45,11 @@ const SHAPE_ID = 'sep-2640-directory-read-result-resources-shape'; const SUBDIR_ID = 'sep-2640-directory-read-subdir-mimetype'; const INVALID_PARAMS_ID = 'sep-2640-directory-read-invalid-params'; const PAGINATION_ID = 'sep-2640-directory-read-pagination'; +const CACHE_ID = 'sep-2640-directory-read-cache-attributes'; +const EMPTY_DIR_ID = 'sep-2640-directory-read-empty-dir'; + +/** How many directories the empty-directory search reads before giving up. */ +const MAX_DIRS_SEARCHED = 32; const ALL_IDS = [ CAPABILITY_ID, @@ -52,12 +57,16 @@ const ALL_IDS = [ SHAPE_ID, SUBDIR_ID, INVALID_PARAMS_ID, - PAGINATION_ID + PAGINATION_ID, + CACHE_ID, + EMPTY_DIR_ID ]; interface DirectoryReadResult { resources?: SkillResource[]; nextCursor?: string; + ttlMs?: unknown; + cacheScope?: unknown; } /** A directory to exercise plus, when known, a non-directory resource under it. */ @@ -136,6 +145,8 @@ export class SkillsDirectoryReadScenario implements ClientScenario { - \`sep-2640-directory-read-subdir-mimetype\` — subdirectory children carry \`inode/directory\` (MUST) - \`sep-2640-directory-read-invalid-params\` — a non-directory URI returns \`-32602\` (MUST) - \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list (single-page is conformant) +- \`sep-2640-directory-read-cache-attributes\` — the result carries \`ttlMs\` and \`cacheScope\` (from 2026-07-28) +- \`sep-2640-directory-read-empty-dir\` — an empty subdirectory reads as \`resources: []\` (SKIPs when there is none) **Gating & discovery**: the checks SKIP when the skills extension or its \`directoryRead\` flag is undeclared. The directory to exercise is discovered dynamically from \`skills/list\` / \`resources/list\` — no fixture URI is hardcoded.`; @@ -193,7 +204,9 @@ export class SkillsDirectoryReadScenario implements ClientScenario { [ PAGINATION_ID, 'nextCursor round-trips per the resources/list contract.' - ] + ], + [CACHE_ID, 'The directory result carries ttlMs and cacheScope.'], + [EMPTY_DIR_ID, 'An empty directory yields an empty resources array.'] ]; for (const [id, desc] of rest) { checks.push( @@ -385,9 +398,126 @@ export class SkillsDirectoryReadScenario implements ClientScenario { ) ); + checks.push(cacheCheck(happy, ctx.specVersion)); + checks.push( + await emptyDirCheck(conn, happy, await otherSkillRoots(conn, target)) + ); + return checks; } finally { await conn.close(); } } } + +const CACHE_TEXT = + 'ReadResourceDirectoryResult extends CacheableResult, so ttlMs and cacheScope are REQUIRED, as they are on resources/list.'; + +/** + * ttlMs and cacheScope on the first page of the directory read. Below + * 2026-07-28 CacheableResult does not exist, so the check SKIPs there, the + * same way the skills/list and skills/get cache checks do. + */ +function cacheCheck( + happy: DirectoryReadResult | undefined, + specVersion: string +): ConformanceCheck { + if (specVersion < '2026-07-28') { + return skillsCheck(CACHE_ID, CACHE_TEXT, 'SKIPPED', { + errorMessage: `not applicable on negotiated protocol ${specVersion}: ttlMs and cacheScope are defined from 2026-07-28` + }); + } + if (happy === undefined) { + return skillsCheck(CACHE_ID, CACHE_TEXT, 'FAILURE', { + errorMessage: 'directory read did not return a result' + }); + } + const missing = [ + happy.ttlMs === undefined && 'ttlMs', + happy.cacheScope === undefined && 'cacheScope' + ] + .filter(Boolean) + .join(' and '); + return skillsCheck( + CACHE_ID, + CACHE_TEXT, + missing ? 'FAILURE' : 'SUCCESS', + missing + ? { + errorMessage: `resources/directory/read result omits ${missing} on protocol ${specVersion}.` + } + : { details: { ttlMs: happy.ttlMs, cacheScope: happy.cacheScope } } + ); +} + +/** The root directory of every listed skill other than the one exercised. */ +async function otherSkillRoots( + conn: Connection, + target: DirectoryTarget +): Promise { + const listed = await skillsListAll(conn); + if ('error' in listed) return []; + return listed.entries + .map((e) => + typeof e.uri === 'string' ? skillRootFromManifestUri(e.uri) : undefined + ) + .filter((r): r is string => r !== undefined && r !== target.dirUri); +} + +/** + * Searches the skill's subdirectories, then the other listed skills' trees, + * breadth first, for one with no + * children, and checks that its read answers `resources: []`. A read that + * answers no `resources` member at all is the failure: a client cannot tell + * that from a broken reply. Having no empty directory is not a defect, so + * that case SKIPs. + */ +async function emptyDirCheck( + conn: Connection, + happy: DirectoryReadResult | undefined, + otherRoots: string[] +): Promise { + const text = 'An empty directory yields an empty resources array.'; + const queue = (happy?.resources ?? []) + .filter((r) => r.mimeType === DIRECTORY_MIME && typeof r.uri === 'string') + .map((r) => r.uri); + queue.push(...otherRoots); + let read = 0; + while (queue.length > 0 && read < MAX_DIRS_SEARCHED) { + const uri = queue.shift()!; + read++; + let res: DirectoryReadResult; + try { + res = await conn.request( + 'resources/directory/read', + { uri } + ); + } catch (e) { + return skillsCheck(EMPTY_DIR_ID, text, 'FAILURE', { + errorMessage: `resources/directory/read on the listed subdirectory ${uri} failed: ${e instanceof Error ? e.message : String(e)}` + }); + } + const children = res.resources; + if (children === undefined || children === null) { + if (res.nextCursor) continue; + return skillsCheck(EMPTY_DIR_ID, text, 'FAILURE', { + errorMessage: `resources/directory/read on ${uri} answered with no resources member; an empty directory yields resources: [].` + }); + } + if (Array.isArray(children) && children.length === 0 && !res.nextCursor) { + return skillsCheck(EMPTY_DIR_ID, text, 'SUCCESS', { + details: { emptyDirectory: uri } + }); + } + if (Array.isArray(children)) { + for (const c of children) { + if (c.mimeType === DIRECTORY_MIME && typeof c.uri === 'string') { + queue.push(c.uri); + } + } + } + } + return skillsCheck(EMPTY_DIR_ID, text, 'SKIPPED', { + errorMessage: `no empty directory found in the ${read} subdirectories read; serving one is optional.` + }); +} diff --git a/src/scenarios/server/skills/enumeration.ts b/src/scenarios/server/skills/enumeration.ts index 68b3eebc..e0a00521 100644 --- a/src/scenarios/server/skills/enumeration.ts +++ b/src/scenarios/server/skills/enumeration.ts @@ -20,7 +20,7 @@ import { isDeepStrictEqual } from 'util'; import { ClientScenario, ConformanceCheck } from '../../../types'; -import type { RunContext } from '../../../connection'; +import { Connection, JsonRpcError, type RunContext } from '../../../connection'; import { SKILLS_EXTENSION_ID, SKILLS_LIST_METHOD, @@ -33,6 +33,7 @@ import { MAX_TOTAL_SIZE_PER_SKILL, FRONTMATTER_RESERVED_PREFIX, JSONRPC_INVALID_PARAMS, + JSONRPC_RESOURCE_NOT_FOUND_2025, type SkillEntry, type SkillResourceEntry, declaredSkillsCapability, @@ -81,7 +82,9 @@ const ENTRY_IDS = [ 'sep-2640-metadata-reserved-prefix', 'sep-2640-name-naming-rules', 'sep-2640-authority-reg-name', - 'sep-2640-names-should-be-unique' + 'sep-2640-names-should-be-unique', + 'sep-2640-resources-nested-completeness', + 'sep-2640-nested-publication-flat' ] as const; const GET_IDS = [ @@ -96,7 +99,9 @@ const GET_IDS = [ const READBACK_IDS = [ 'sep-2640-skillmd-required', 'sep-2640-skillmd-frontmatter', - 'sep-2640-entry-frontmatter-identical' + 'sep-2640-entry-frontmatter-identical', + 'sep-2640-error-standard-codes', + 'sep-2640-error-informative-messages' ] as const; const ALL_CHECK_IDS = [ @@ -150,6 +155,8 @@ export class SkillsEnumerationScenario implements ClientScenario { - \`sep-2640-limit-resources-per-skill\` / \`sep-2640-limit-total-size\` — 512 entries, 16 MiB - \`sep-2640-metadata-reserved-prefix\` — frontmatter \`metadata\` keys under \`io.modelcontextprotocol/\` are reserved - \`sep-2640-skills-get-*\` — \`skills/get\` returns a list-shaped entry, carries no cursor, and answers \`-32602\` for an unknown URI +- \`sep-2640-resources-nested-completeness\` / \`sep-2640-nested-publication-flat\` — an enclosing skill lists a nested skill's files, and no entry embeds another (SKIP without nesting) +- \`sep-2640-error-standard-codes\` / \`sep-2640-error-informative-messages\` — an unserved file under a skill answers \`-32602\` with a message naming the cause **Discovery is dynamic**: an undeclared extension SKIPs everything; an empty or partial listing is permitted and SKIPs the entry-level checks.`; @@ -414,8 +421,10 @@ export class SkillsEnumerationScenario implements ClientScenario { } checks.push(...entryChecks(entries)); + checks.push(...nestedChecks(entries)); checks.push(...(await getChecks(conn, entries, ctx.specVersion))); checks.push(...(await readbackChecks(conn, entries))); + checks.push(...(await errorChecks(conn, entries, ctx.specVersion))); return checks; } finally { @@ -1213,3 +1222,252 @@ async function getChecks( return checks; } + +/** + * The listed pairs where one skill sits inside another: the inner skill's root + * directory is a descendant of the outer skill's root. + */ +function nestedPairs( + entries: SkillEntry[] +): Array<{ outer: SkillEntry; inner: SkillEntry }> { + const rooted = entries + .map((e) => ({ + e, + root: + typeof e.uri === 'string' ? skillRootFromManifestUri(e.uri) : undefined + })) + .filter((x): x is { e: SkillEntry; root: string } => x.root !== undefined); + const pairs: Array<{ outer: SkillEntry; inner: SkillEntry }> = []; + for (const o of rooted) { + for (const i of rooted) { + if (i.root.startsWith(`${o.root}/`)) + pairs.push({ outer: o.e, inner: i.e }); + } + } + return pairs; +} + +/** An object shaped like a skill entry, which a flat listing never embeds. */ +function looksLikeEntry(value: unknown): boolean { + return ( + isSettingsObject(value) && + typeof value.uri === 'string' && + isSettingsObject(value.frontmatter) + ); +} + +/** + * Nested skills: completeness of the enclosing skill's `resources`, and a + * listing that stays flat. Nesting is optional, so a listing without a nested + * pair SKIPs both. + */ +function nestedChecks(entries: SkillEntry[]): ConformanceCheck[] { + const pairs = nestedPairs(entries); + const completenessText = + "Completeness extends to nested skills: the enclosing skill's resources lists the nested skill's files too."; + const flatText = + 'A nested skill is published like any other, and nothing in the listing marks nesting.'; + if (pairs.length === 0) { + const reason = + 'No listed skill sits inside another, so there is no nesting to check. Nesting is optional.'; + return [ + skillsCheck( + 'sep-2640-resources-nested-completeness', + completenessText, + 'SKIPPED', + { + errorMessage: reason + } + ), + skillsCheck('sep-2640-nested-publication-flat', flatText, 'SKIPPED', { + errorMessage: reason + }) + ]; + } + + // === resources-nested-completeness === + // The inner skill's files, as its own entry lists them, must all appear in + // the outer skill's resources. A "dynamic" outer publishes no manifest, so + // there is nothing to be complete. + const missing: string[] = []; + let graded = 0; + for (const { outer, inner } of pairs) { + const outerArr = resourcesArray(outer); + if (!outerArr) continue; + graded++; + const have = new Set( + outerArr + .map((r) => r.uri) + .filter((u): u is string => typeof u === 'string') + ); + const innerArr = resourcesArray(inner); + const want = innerArr + ? innerArr + .map((r) => r.uri) + .filter((u): u is string => typeof u === 'string') + : [inner.uri as string]; + for (const uri of want) { + if (!have.has(uri)) { + missing.push( + `${String(outer.uri)} does not list ${uri}, a file of the nested skill ${String(inner.uri)}` + ); + } + } + } + const checks: ConformanceCheck[] = []; + checks.push( + graded === 0 + ? skillsCheck( + 'sep-2640-resources-nested-completeness', + completenessText, + 'SKIPPED', + { + errorMessage: + 'Every enclosing skill declares "resources": "dynamic", which publishes no manifest to be complete.' + } + ) + : skillsCheck( + 'sep-2640-resources-nested-completeness', + completenessText, + missing.length === 0 ? 'SUCCESS' : 'FAILURE', + missing.length === 0 + ? { details: { nestedPairs: pairs.length } } + : { errorMessage: joinErrs(missing) } + ) + ); + + // === nested-publication-flat === + // A flat listing carries the nested skill as its own top-level entry. An + // entry that embeds another entry, alone or in an array, marks nesting. + const embedded: string[] = []; + entries.forEach((e, i) => { + for (const [key, value] of Object.entries(e)) { + if (key === 'frontmatter' || key === 'resources') continue; + const values = Array.isArray(value) ? value : [value]; + if (values.some(looksLikeEntry)) { + embedded.push( + `${entryLabel(e, i)}: field "${key}" embeds a skill entry` + ); + } + } + }); + checks.push( + skillsCheck( + 'sep-2640-nested-publication-flat', + flatText, + embedded.length === 0 ? 'SUCCESS' : 'FAILURE', + embedded.length === 0 + ? { details: { nestedPairs: pairs.length } } + : { errorMessage: joinErrs(embedded) } + ) + ); + return checks; +} + +/** A file name no conformant server serves, for the unserved-file probe. */ +const UNSERVED_FILE = 'mcp-conformance-unserved-9f3a2b.md'; + +/** Bare code names, which describe the code rather than the cause. */ +const BARE_ERROR_MESSAGES = new Set([ + '', + 'error', + 'invalid params', + 'internal error', + 'resource not found' +]); + +/** + * `resources/read` of a file the server does not serve, under a listed skill's + * root. The page names `-32602`. The 2025-11-25 base spec named `-32002` for an + * unknown resource, so that code also passes on that wire. + */ +async function errorChecks( + conn: Connection, + entries: SkillEntry[], + specVersion: string +): Promise { + const codesText = + 'resources/read of a skill file the server does not serve returns -32602 (Invalid params), per the base Resources specification.'; + const messageText = + 'Servers SHOULD provide informative error messages to describe the cause of errors.'; + const root = entries + .map((e) => + typeof e.uri === 'string' ? skillRootFromManifestUri(e.uri) : undefined + ) + .find((r): r is string => r !== undefined); + if (!root) { + const reason = `No listed entry has a uri ending in /${SKILL_MANIFEST_FILENAME}, so there is no skill directory to probe.`; + return [ + skillsCheck('sep-2640-error-standard-codes', codesText, 'SKIPPED', { + errorMessage: reason + }), + skillsCheck( + 'sep-2640-error-informative-messages', + messageText, + 'SKIPPED', + { errorMessage: reason } + ) + ]; + } + + const probe = `${root}/${UNSERVED_FILE}`; + const accepted = + specVersion >= '2026-07-28' + ? [JSONRPC_INVALID_PARAMS] + : [JSONRPC_INVALID_PARAMS, JSONRPC_RESOURCE_NOT_FOUND_2025]; + let error: JsonRpcError | undefined; + let other = ''; + try { + await conn.request('resources/read', { uri: probe }); + other = `resources/read of ${probe} succeeded, though the skill lists no such file`; + } catch (e) { + if (e instanceof JsonRpcError) error = e; + else + other = `resources/read of ${probe} failed without a JSON-RPC error: ${e instanceof Error ? e.message : String(e)}`; + } + + const checks: ConformanceCheck[] = []; + const codeOk = error !== undefined && accepted.includes(error.code); + checks.push( + skillsCheck( + 'sep-2640-error-standard-codes', + codesText, + codeOk ? 'SUCCESS' : 'FAILURE', + codeOk + ? { details: { probedUri: probe, code: error!.code } } + : { + errorMessage: error + ? `resources/read of ${probe} returned ${error.code} (${error.message}); expected ${accepted.join(' or ')} on protocol ${specVersion}.` + : other + } + ) + ); + + if (!error) { + checks.push( + skillsCheck( + 'sep-2640-error-informative-messages', + messageText, + 'SKIPPED', + { + errorMessage: `${other}, so there is no error message to inspect.` + } + ) + ); + return checks; + } + const bare = BARE_ERROR_MESSAGES.has(error.message.trim().toLowerCase()); + checks.push( + skillsCheck( + 'sep-2640-error-informative-messages', + messageText, + bare ? 'WARNING' : 'SUCCESS', + bare + ? { + errorMessage: `the error for ${probe} says only ${JSON.stringify(error.message)}, which names the code rather than the cause.` + } + : { details: { message: error.message } } + ) + ); + return checks; +} diff --git a/src/scenarios/server/skills/helpers.ts b/src/scenarios/server/skills/helpers.ts index df578a44..72c6da67 100644 --- a/src/scenarios/server/skills/helpers.ts +++ b/src/scenarios/server/skills/helpers.ts @@ -47,6 +47,8 @@ export const MAX_TOTAL_SIZE_PER_SKILL = 16 * 1024 * 1024; // 16 MiB export const JSONRPC_METHOD_NOT_FOUND = -32601; export const JSONRPC_INVALID_PARAMS = -32602; +/** The 2025-11-25 base spec's code for an unknown resource, replaced by -32602 from 2026-07-28. */ +export const JSONRPC_RESOURCE_NOT_FOUND_2025 = -32002; export const SEP_2640_REF: SpecReference = { id: 'SEP-2640', diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml index 9ecfdba2..c7c7e7e6 100644 --- a/src/seps/sep-2640.yaml +++ b/src/seps/sep-2640.yaml @@ -73,11 +73,12 @@ # This file declares 89 checks plus 11 excluded rows. Of the uncovered # sentences, four are base-protocol rules (below), several restate a declared # row in Security Considerations, and the rest are MAY permissions with -# nothing to check. Of the 89 checks, 46 are emitted by a scenario: 42 by the -# three server scenarios and 4 by the two client scenarios. The other 43 are -# host obligations (approval binding, cache isolation, context handling) that -# no current scenario can observe. Nothing recomputes these counts, so treat -# them as a claim with a date. +# nothing to check. Of the 89 checks, 59 are emitted by a scenario: 48 by the +# three server scenarios and 11 by the six client scenarios (counted +# 2026-11-16). The other 30 are host obligations (approval binding, cache +# isolation, context handling, what the model is shown) that no scenario can +# observe on the wire, plus MAY permissions with nothing to fail. Nothing +# recomputes these counts, so treat them as a claim with a date. # # deliberately not declared here: `resultType` and request `_meta`. The page # says `resultType` MUST be `"complete"` on each of its three results, and @@ -88,13 +89,14 @@ # base-protocol yamls. # # backing_scenarios: three server ClientScenarios under -# src/scenarios/server/skills/ and two client scenarios under +# src/scenarios/server/skills/ and six client scenarios under # src/scenarios/client/skills/ emit the check IDs below (a row is "tested" # once a scenario emits its check ID; see src/traceability/): # enumeration.ts (sep-2640-skills-enumeration) — the capability rows, the # sep-2640-skills-list-* rows, the entry-schema and resources rows, the two -# limit rows, and the sep-2640-skills-get-* rows. Replaces the retired -# index.ts, which tested `skill://index.json`. +# limit rows, the sep-2640-skills-get-* rows, the two nested-skill rows, +# and the two error-handling rows. Replaces the retired index.ts, which +# tested `skill://index.json`. # manifest.ts (sep-2640-skills-manifest) — sep-2640-skillmd-mimetype, # sep-2640-skillmd-metadata-name, sep-2640-skillmd-metadata-description, # sep-2640-final-segment-equals-name, sep-2640-meta-prefix. @@ -105,9 +107,17 @@ # verification.ts (sep-2640-client-verify-digest, -verify-size, # -verify-frontmatter) — sep-2640-host-verify-digest, # sep-2640-host-size-mismatch-failure, sep-2640-host-frontmatter-comparison. +# hosting.ts (sep-2640-client-load-by-uri, -invalid-entry, +# -directory-unlisted-child, -two-origins) — sep-2640-host-load-by-uri, +# sep-2640-host-no-empty-listing-assumption, +# sep-2640-resources-invalid-entry, +# sep-2640-directory-read-not-manifest-extension, +# sep-2640-host-skill-identity-pair, sep-2640-host-cross-origin-no-shadow, +# sep-2640-host-reads-bound-to-origin. The two-origins scenario runs two +# mock servers and hands the client the second one's URL in +# MCP_CONFORMANCE_CONTEXT. # The remaining rows are host-internal or off-wire (approval binding, cache -# isolation, context handling) and stay traceability-only. More client -# scenarios would be needed to exercise them. +# isolation, context handling) and stay traceability-only. sep: 2640 spec_url: https://skills.extensions.modelcontextprotocol.io/specification/beta/skills requirements: