Repository navigation
feat(sep-3415): Conformance suite for triggers and events #521
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
cb0f5af
1207bf8
14457c1
0adb7ce
6d76f93
0ed9009
c465929
b7fcf3f
d77741d
0ffd1ce
d3d26e0
5befc60
f9eb9a4
6be2ca1
80d6528
77c10e0
3089d62
5aceffa
3051ea2
e18b608
51f85d7
07890dc
8edc0cc
df035e5
863b20b
45e4ede
2323122
02271a0
0e58df1
b4ce577
85ce513
d3756d7
43b835d
c377770
24e1f1b
57212b0
4a73fc6
b33cb4d
2eb3b50
e58b9ed
23c7c2d
9c67e6d
b7bac9e
76fef32
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,6 +37,26 @@ describe('buildStandardHeaders', () => { | |
| ).toBe('file:///a.txt'); | ||
| }); | ||
|
|
||
| test('sets Mcp-Name from params.name on events/* except events/list (SEP-3415)', () => { | ||
| for (const method of [ | ||
| 'events/poll', | ||
| 'events/stream', | ||
| 'events/subscribe', | ||
| 'events/unsubscribe' | ||
| ]) { | ||
| expect(buildStandardHeaders(method, { name: 'tick' })['Mcp-Name']).toBe( | ||
| 'tick' | ||
| ); | ||
| } | ||
| expect(buildStandardHeaders('events/list', {})['Mcp-Name']).toBeUndefined(); | ||
| }); | ||
|
|
||
| test('wraps a non-ASCII event name in the SEP-2243 Base64 sentinel', () => { | ||
| expect( | ||
| buildStandardHeaders('events/poll', { name: 'café' })['Mcp-Name'] | ||
| ).toBe(`=?base64?${Buffer.from('café').toString('base64')}?=`); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
| }); | ||
|
|
||
| test('overrides replace defaults case-insensitively', () => { | ||
| const headers = buildStandardHeaders('tools/list', undefined, { | ||
| headers: { 'mcp-protocol-version': '2025-06-18' } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -60,7 +60,8 @@ let nextRequestId = 1; | |
| /** | ||
| * The `Mcp-Name` source field per SEP-2243: `params.name` for tools/call and | ||
| * prompts/get, `params.uri` for resources/read, `params.taskId` for the | ||
| * SEP-2663 tasks methods (`tasks/get`, `tasks/update`, `tasks/cancel`). | ||
| * SEP-2663 tasks methods (`tasks/get`, `tasks/update`, `tasks/cancel`), and | ||
| * `params.name` for the SEP-3415 events methods other than `events/list`. | ||
| * Absent otherwise. | ||
| */ | ||
| export function mcpNameForRequest( | ||
|
|
@@ -80,9 +81,37 @@ export function mcpNameForRequest( | |
| ) { | ||
| return typeof params?.taskId === 'string' ? params.taskId : undefined; | ||
| } | ||
| if (EVENTS_NAMED_METHODS.has(method)) { | ||
| return typeof params?.name === 'string' | ||
| ? encodeMcpHeaderValue(params.name) | ||
| : undefined; | ||
| } | ||
| return undefined; | ||
| } | ||
|
|
||
| /** | ||
| * The SEP-3415 methods that carry `Mcp-Name: <params.name>` (its "Streamable | ||
| * HTTP: Routing Headers" section). `events/list` has no `params.name` and is | ||
| * not one of them. | ||
| */ | ||
| const EVENTS_NAMED_METHODS = new Set([ | ||
| 'events/poll', | ||
| 'events/stream', | ||
| 'events/subscribe', | ||
| 'events/unsubscribe' | ||
| ]); | ||
|
|
||
| /** | ||
| * SEP-2243's header encoding: a value that is not plain printable ASCII, or | ||
| * that has leading or trailing whitespace, travels as `=?base64?{utf8}?=`. | ||
| */ | ||
| export function encodeMcpHeaderValue(value: string): string { | ||
| const plain = /^[\x20-\x7e]*$/.test(value) && value.trim() === value; | ||
| return plain | ||
| ? value | ||
| : `=?base64?${Buffer.from(value, 'utf8').toString('base64')}?=`; | ||
| } | ||
|
Comment on lines
+104
to
+113
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Missing the sentinel check mentioned here: https://modelcontextprotocol.io/seps/2243-http-standardization#value-encoding |
||
|
|
||
| /** | ||
| * Build the conformant header set for a stateless request: Content-Type, | ||
| * Accept (both content types), MCP-Protocol-Version, Mcp-Method and (when the | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| import { describe, test, expect } from 'vitest'; | ||
| import type { ConformanceCheck } from '../types'; | ||
| import { untestableCheck, notTestable } from '../scenarios/untestable'; | ||
| import { formatTally, isUnverified, tallyChecks } from './summary'; | ||
|
|
||
| const check = ( | ||
| id: string, | ||
| status: ConformanceCheck['status'], | ||
| extra: Partial<ConformanceCheck> = {} | ||
| ): ConformanceCheck => ({ | ||
| id, | ||
| name: id, | ||
| description: id, | ||
| status, | ||
| timestamp: '2026-09-24T00:00:00.000Z', | ||
| specReferences: [], | ||
| ...extra | ||
| }); | ||
|
|
||
| const ref = [{ specVersion: 'draft', section: 'x' }] as never; | ||
|
|
||
| describe('tallyChecks', () => { | ||
| test('counts the same four numbers the runners counted before', () => { | ||
| const tally = tallyChecks([ | ||
| check('a', 'SUCCESS'), | ||
| check('b', 'SUCCESS'), | ||
| check('c', 'FAILURE'), | ||
| check('d', 'WARNING'), | ||
| check('e', 'SKIPPED') | ||
| ]); | ||
| expect(tally).toMatchObject({ | ||
| passed: 2, | ||
| failed: 1, | ||
| warnings: 1, | ||
| denominator: 3 | ||
| }); | ||
| }); | ||
|
|
||
| test('unverified is a subset of failed and warnings, never an addition', () => { | ||
| const tally = tallyChecks([ | ||
| check('violated', 'FAILURE', { errorMessage: 'cursor was 42' }), | ||
| untestableCheck('missing', 'missing', 'd', 'no control', ref), | ||
| untestableCheck('soft', 'soft', 'd', 'no control', ref, 'WARNING') | ||
| ]); | ||
| expect(tally.failed).toBe(2); | ||
| expect(tally.warnings).toBe(1); | ||
| expect(tally.unverified).toBe(2); | ||
| // The verdict a runner keys its exit code off is unchanged by the split. | ||
| expect(tally.failed + tally.warnings).toBeGreaterThan(0); | ||
| }); | ||
|
|
||
| test('a scenario that formats its own message still counts as unverified', () => { | ||
| const own = check('own', 'FAILURE', { | ||
| errorMessage: notTestable('the fixture exposes no control'), | ||
| details: {} | ||
| }); | ||
| expect(isUnverified(own)).toBe(true); | ||
| expect(tallyChecks([own]).unverified).toBe(1); | ||
| }); | ||
|
|
||
| test('a skipped check is neither failed nor unverified', () => { | ||
| const skipped = check('skip', 'SKIPPED', { | ||
| errorMessage: notTestable('does not apply') | ||
| }); | ||
| expect(isUnverified(skipped)).toBe(false); | ||
| expect(tallyChecks([skipped])).toMatchObject({ | ||
| failed: 0, | ||
| unverified: 0, | ||
| denominator: 0 | ||
| }); | ||
| }); | ||
| }); | ||
|
|
||
| describe('formatTally', () => { | ||
| test('prints what it always printed when nothing is unverified', () => { | ||
| const line = formatTally({ | ||
| passed: 9, | ||
| failed: 1, | ||
| warnings: 0, | ||
| denominator: 10, | ||
| unverified: 0 | ||
| }); | ||
| expect(line).toBe('Passed: 9/10, 1 failed, 0 warnings'); | ||
| }); | ||
|
|
||
| test('says how much of a red run was never checked', () => { | ||
| const line = formatTally({ | ||
| passed: 2, | ||
| failed: 27, | ||
| warnings: 0, | ||
| denominator: 29, | ||
| unverified: 27 | ||
| }); | ||
| expect(line).toContain('27 failed'); | ||
| expect(line).toContain('27 of those unverified, not violated'); | ||
| }); | ||
| }); |
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Inclined to say this should be pulled out to a dedicated PR and stacked under this one, will leave that decision to @felixweinberger, though. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /** | ||
| * One tally for every runner's results line. | ||
| * | ||
| * The `server`, `client` and `authorization-server` runners each counted the | ||
| * same four numbers from the same filters, so a fifth number had to be added in | ||
| * three places or in none. | ||
| * | ||
| * That fifth number is what a reader of a red run most wants: how much of it was | ||
| * a requirement being violated, and how much was a requirement that could not be | ||
| * checked at all. Those are different facts with the same status, because a | ||
| * missing prerequisite reports as a failure on purpose (issue #248) — SKIPPED is | ||
| * excluded from counts, exit codes and the expected-failures baseline, so a | ||
| * skipped row is one nobody can burn down. Splitting them in the output costs | ||
| * nothing and changes no verdict: `unverified` counts rows already inside | ||
| * `failed` and `warnings`, so exit codes stay exactly as they were. | ||
| */ | ||
|
|
||
| import type { ConformanceCheck } from '../types'; | ||
| import { notTestable } from '../scenarios/untestable'; | ||
|
|
||
| /** The prefix `notTestable()` writes, for checks that carry no details flag. */ | ||
| const NOT_TESTABLE_PREFIX = notTestable('').trim(); | ||
|
|
||
| export interface CheckTally { | ||
| /** Checks that passed. */ | ||
| passed: number; | ||
| /** Checks with status FAILURE, including ones that could not be exercised. */ | ||
| failed: number; | ||
| /** Checks with status WARNING, including ones that could not be exercised. */ | ||
| warnings: number; | ||
| /** SUCCESS + FAILURE, the population the pass ratio is quoted against. */ | ||
| denominator: number; | ||
| /** | ||
| * Of `failed` + `warnings`, how many report a missing prerequisite rather | ||
| * than a violated requirement. A subset, never an addition. | ||
| */ | ||
| unverified: number; | ||
| } | ||
|
|
||
| /** | ||
| * Whether a check reports a prerequisite it could not satisfy. | ||
| * | ||
| * Either signal counts: `untestableCheck()` sets `details.untestable`, and a | ||
| * scenario that formats its own message with `notTestable()` carries only the | ||
| * prefix. Reading both means a scenario cannot fall out of the tally by | ||
| * building its check the other way. | ||
| */ | ||
| export function isUnverified(check: ConformanceCheck): boolean { | ||
| if (check.status !== 'FAILURE' && check.status !== 'WARNING') return false; | ||
| if (check.details?.untestable === true) return true; | ||
| return (check.errorMessage ?? '').startsWith(NOT_TESTABLE_PREFIX); | ||
| } | ||
|
|
||
| export function tallyChecks(checks: ConformanceCheck[]): CheckTally { | ||
| return { | ||
| passed: checks.filter((c) => c.status === 'SUCCESS').length, | ||
| failed: checks.filter((c) => c.status === 'FAILURE').length, | ||
| warnings: checks.filter((c) => c.status === 'WARNING').length, | ||
| denominator: checks.filter( | ||
| (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' | ||
| ).length, | ||
| unverified: checks.filter(isUnverified).length | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * The results line. The unverified clause is appended only when there is one, | ||
| * so a run with nothing unverified prints exactly what it always printed. | ||
| */ | ||
| export function formatTally(tally: CheckTally): string { | ||
| const base = `Passed: ${tally.passed}/${tally.denominator}, ${tally.failed} failed, ${tally.warnings} warnings`; | ||
| return tally.unverified > 0 | ||
| ? `${base} (${tally.unverified} of those unverified, not violated)` | ||
| : base; | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we need to add any tests for
Mcp-Methodas well, or do we have something that auto-fails on that for everything already?