From 71e15ac145be5c44798ea1d2f370f7d2b1d742a3 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Fri, 13 Nov 2026 12:30:14 -0800 Subject: [PATCH] feat: report expected-vs-emitted check coverage per scenario (#451) The traceability run now records each scenario's counted check IDs (SUCCESS, FAILURE, WARNING) under a new scenarios key in traceability.json. After a scenario runs, the runner compares what it emitted against that baseline. It prints the count and the missing IDs, adds a coverage suffix to the suite summary, and writes coverage.json next to checks.json. Coverage is advisory and never changes the exit code. A run away from the scenario's default spec version gets no baseline, so it cannot show a false gap. --- src/coverage.test.ts | 107 +++++++++++++++++++++++++++++ src/coverage.ts | 122 +++++++++++++++++++++++++++++++++ src/index.ts | 25 +++++-- src/runner/client.ts | 24 ++++++- src/runner/server.ts | 42 +++++++++--- src/traceability/index.test.ts | 74 +++++++++++++++++++- src/traceability/index.ts | 95 ++++++++++++++++++++++++- src/traceability/types.ts | 22 ++++-- 8 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 src/coverage.test.ts create mode 100644 src/coverage.ts diff --git a/src/coverage.test.ts b/src/coverage.test.ts new file mode 100644 index 00000000..434c7611 --- /dev/null +++ b/src/coverage.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { + computeCoverage, + coverageSuffix, + expectedFor, + formatCoverage +} from './coverage'; +import type { ConformanceCheck, CheckStatus } from './types'; +import type { TraceabilityManifest } from './traceability/types'; + +function check(id: string, status: CheckStatus = 'SUCCESS'): ConformanceCheck { + return { + id, + name: id, + description: id, + status, + timestamp: '2026-01-01T00:00:00.000Z' + }; +} + +const EXPECTED = ['a', 'b', 'c', 'd', 'wire-schema-valid']; + +describe('computeCoverage', () => { + it('reports every expected check when the run is complete', () => { + const c = computeCoverage( + EXPECTED, + EXPECTED.map((id) => check(id)) + ); + expect(c).toEqual({ expected: 5, emitted: 5, missing: [] }); + expect(coverageSuffix(c)).toBe(''); + }); + + it('shows a thin run that passes every check it emitted', () => { + // Every emitted check is green, so pass/fail reads as a clean pass. + const checks = [check('b'), check('wire-schema-valid')]; + const c = computeCoverage(EXPECTED, checks)!; + expect(checks.every((x) => x.status === 'SUCCESS')).toBe(true); + expect(c).toEqual({ expected: 5, emitted: 2, missing: ['a', 'c', 'd'] }); + expect(coverageSuffix(c)).toBe(' (coverage 2/5)'); + }); + + it('shows how much of the scenario a setup failure skipped', () => { + const c = computeCoverage(EXPECTED, [check('a', 'FAILURE')])!; + expect(c.emitted).toBe(1); + expect(c.missing).toEqual(['b', 'c', 'd', 'wire-schema-valid']); + }); + + it('does not count INFO or SKIPPED rows as emitted', () => { + const c = computeCoverage( + ['a', 'b'], + [check('a', 'INFO'), check('b', 'SKIPPED')] + )!; + expect(c.emitted).toBe(0); + }); + + it('counts WARNING as emitted', () => { + expect(computeCoverage(['a'], [check('a', 'WARNING')])!.missing).toEqual( + [] + ); + }); + + it('returns null without a baseline', () => { + expect(computeCoverage(undefined, [check('a')])).toBeNull(); + expect(computeCoverage([], [check('a')])).toBeNull(); + }); +}); + +describe('expectedFor', () => { + const manifest: TraceabilityManifest = { + schemaVersion: 1, + docs: '', + source: 'typescript-sdk@test', + seps: {}, + scenarios: { + 'server:tools-list': { expected: ['x'] }, + 'client:auth/dpop': { expected: ['y'] } + } + }; + + it('keys by CLI mode and scenario name', () => { + expect(expectedFor('server', 'tools-list', true, manifest)).toEqual(['x']); + expect(expectedFor('client', 'auth/dpop', true, manifest)).toEqual(['y']); + expect(expectedFor('client', 'tools-list', true, manifest)).toBeUndefined(); + }); + + it('gives no baseline away from the default spec version', () => { + expect( + expectedFor('server', 'tools-list', false, manifest) + ).toBeUndefined(); + }); + + it('gives no baseline for a manifest built before scenarios existed', () => { + const old = { ...manifest, scenarios: undefined }; + expect(expectedFor('server', 'tools-list', true, old)).toBeUndefined(); + }); +}); + +describe('formatCoverage', () => { + it('lists missing IDs and caps the list', () => { + const missing = Array.from({ length: 12 }, (_, i) => `m${i}`); + const line = formatCoverage({ expected: 15, emitted: 3, missing }); + expect(line).toMatch(/^Coverage: 3\/15 expected checks emitted\. /); + expect(line).toContain('m9'); + expect(line).not.toContain('m10'); + expect(line).toMatch(/and 2 more$/); + }); +}); diff --git a/src/coverage.ts b/src/coverage.ts new file mode 100644 index 00000000..c1522cb6 --- /dev/null +++ b/src/coverage.ts @@ -0,0 +1,122 @@ +/** + * Expected-vs-emitted check coverage per scenario (issue #451). + * + * A scenario can pass while emitting only a fraction of its checks, for + * example when the implementation under test fails early and the checks that + * would have run never fire. Pass/fail cannot show that. This module compares + * the check IDs a run emitted against the IDs the same scenario emitted when + * the traceability workflow ran it against the reference SDK, and reports + * the ones that are missing. + * + * The baseline lives in src/seps/traceability.json under `scenarios`, so SDKs + * do not keep their own per-scenario snapshots. Coverage is advisory: it is + * printed and written to coverage.json, and it never changes the exit code. + */ +import { promises as fs } from 'fs'; +import path from 'path'; +import type { ConformanceCheck } from './types'; +import type { TraceabilityManifest } from './traceability/types'; +import manifestJson from './seps/traceability.json'; + +/** Which CLI mode ran the scenario: `client` tests a client, `server` a server. */ +export type Side = 'client' | 'server'; + +/** Statuses that mean the check actually ran. INFO and SKIPPED do not count. */ +const COUNTED = new Set(['SUCCESS', 'FAILURE', 'WARNING']); + +export interface Coverage { + /** Check IDs the reference SDK emitted for this scenario. */ + expected: number; + /** How many of those this run emitted. */ + emitted: number; + /** Expected check IDs this run did not emit, sorted. */ + missing: string[]; +} + +export function scenarioKey(side: Side, scenario: string): string { + return `${side}:${scenario}`; +} + +export function countedIds(checks: ConformanceCheck[]): Set { + return new Set(checks.filter((c) => COUNTED.has(c.status)).map((c) => c.id)); +} + +/** Pure: null when there is no baseline, or the baseline is empty. */ +export function computeCoverage( + expected: string[] | undefined, + checks: ConformanceCheck[] +): Coverage | null { + if (!expected || expected.length === 0) return null; + const emitted = countedIds(checks); + const missing = expected.filter((id) => !emitted.has(id)).sort(); + return { + expected: expected.length, + emitted: expected.length - missing.length, + missing + }; +} + +const bundled = manifestJson as unknown as TraceabilityManifest; + +/** + * The reference run uses each scenario's default spec version. A run at any + * other version can legitimately emit a different set, so it gets no baseline + * rather than a false gap. + */ +export function expectedFor( + side: Side, + scenario: string, + atDefaultSpecVersion: boolean, + manifest: TraceabilityManifest = bundled +): string[] | undefined { + if (!atDefaultSpecVersion) return undefined; + return manifest.scenarios?.[scenarioKey(side, scenario)]?.expected; +} + +export function scenarioCoverage( + side: Side, + scenario: string, + checks: ConformanceCheck[], + atDefaultSpecVersion: boolean +): Coverage | null { + return computeCoverage( + expectedFor(side, scenario, atDefaultSpecVersion), + checks + ); +} + +const MAX_LISTED = 10; + +/** One line for the per-scenario results, plus the missing IDs if any. */ +export function formatCoverage(c: Coverage): string { + const head = `Coverage: ${c.emitted}/${c.expected} expected checks emitted`; + if (c.missing.length === 0) return head; + const listed = c.missing.slice(0, MAX_LISTED).join(', '); + const more = + c.missing.length > MAX_LISTED + ? `, and ${c.missing.length - MAX_LISTED} more` + : ''; + return `${head}. Not emitted: ${listed}${more}`; +} + +/** Suffix for a summary line. Empty when coverage is complete or unknown. */ +export function coverageSuffix(c: Coverage | null | undefined): string { + if (!c || c.missing.length === 0) return ''; + return ` (coverage ${c.emitted}/${c.expected})`; +} + +export async function writeCoverage( + resultDir: string, + side: Side, + scenario: string, + c: Coverage +): Promise { + await fs.writeFile( + path.join(resultDir, 'coverage.json'), + JSON.stringify( + { scenario, side, baseline: bundled.source, ...c }, + null, + 2 + ) + '\n' + ); +} diff --git a/src/index.ts b/src/index.ts index b376644e..54539480 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ #!/usr/bin/env node import { Command } from 'commander'; +import { Coverage, coverageSuffix } from './coverage'; import { ZodError } from 'zod'; import { promises as fs } from 'fs'; import { @@ -337,6 +338,7 @@ program scenario: scenarioName, checks: result.checks, skipped: result.skipped, + coverage: result.coverage, error: null }; } catch (error) { @@ -354,6 +356,7 @@ program } ], skipped: undefined, + coverage: undefined, error }; } @@ -393,7 +396,8 @@ program const status = failed === 0 && warnings === 0 ? '✓' : '✗'; const warningStr = warnings > 0 ? `, ${warnings} warnings` : ''; console.log( - `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` + `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` + + coverageSuffix(result.coverage) ); if (verbose && failed > 0) { @@ -492,7 +496,8 @@ program result.checks, verbose, result.clientOutput, - result.allowClientError + result.allowClientError, + result.coverage ); if (options.expectedFailures) { @@ -603,7 +608,8 @@ program const { failed } = printServerResults( result.checks, result.scenarioDescription, - verbose + verbose, + result.coverage ); if (options.expectedFailures) { @@ -662,8 +668,11 @@ program `Running ${selection} (${scenarios.length} scenarios) against ${validated.url}\n` ); - const allResults: { scenario: string; checks: ConformanceCheck[] }[] = - []; + const allResults: { + scenario: string; + checks: ConformanceCheck[]; + coverage?: Coverage | null; + }[] = []; for (const scenarioName of scenarios) { console.log(`\n=== Running scenario: ${scenarioName} ===`); @@ -682,7 +691,11 @@ program timeout ) ); - allResults.push({ scenario: scenarioName, checks: result.checks }); + allResults.push({ + scenario: scenarioName, + checks: result.checks, + coverage: result.coverage + }); } catch (error) { console.error(`Failed to run scenario ${scenarioName}:`, error); allResults.push({ diff --git a/src/runner/client.ts b/src/runner/client.ts index af10657f..2214163c 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,4 +1,10 @@ import { spawn } from 'child_process'; +import { + Coverage, + formatCoverage, + scenarioCoverage, + writeCoverage +} from '../coverage'; import { promises as fs } from 'fs'; import path from 'path'; import { @@ -159,6 +165,7 @@ export async function runConformanceTest( resultDir?: string; allowClientError?: boolean; skipped?: boolean; + coverage?: Coverage | null; }> { let resultDir: string | undefined; @@ -222,11 +229,21 @@ export async function runConformanceTest( const checks = scenario.getChecks(); checks.push(...wireSchemaChecks(resolvedVersion)); + const coverage = scenarioCoverage( + 'client', + scenarioName, + checks, + resolvedVersion === resolveScenarioSpecVersion(scenario.source, undefined) + ); + if (resultDir) { await fs.writeFile( path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2) ); + if (coverage) { + await writeCoverage(resultDir, 'client', scenarioName, coverage); + } await fs.writeFile( path.join(resultDir, 'stdout.txt'), @@ -245,7 +262,8 @@ export async function runConformanceTest( checks, clientOutput, resultDir, - allowClientError: scenario.allowClientError + allowClientError: scenario.allowClientError, + coverage }; } finally { await scenario.stop(); @@ -256,7 +274,8 @@ export function printClientResults( checks: ConformanceCheck[], verbose: boolean = false, clientOutput?: ClientExecutionResult, - allowClientError: boolean = false + allowClientError: boolean = false, + coverage?: Coverage | null ): { passed: number; failed: number; @@ -295,6 +314,7 @@ export function printClientResults( console.error( `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` ); + if (coverage) console.error(formatCoverage(coverage)); if (clientTimedOut) { console.error(`\n⚠️ CLIENT TIMED OUT - Test incomplete`); diff --git a/src/runner/server.ts b/src/runner/server.ts index 21e80ec4..adf9d0b0 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -1,4 +1,11 @@ import { promises as fs } from 'fs'; +import { + Coverage, + coverageSuffix, + formatCoverage, + scenarioCoverage, + writeCoverage +} from '../coverage'; import path from 'path'; import { ConformanceCheck, @@ -85,6 +92,7 @@ export async function runServerConformanceTest( resultDir?: string; scenarioDescription: string; skipped?: boolean; + coverage?: Coverage | null; }> { let resultDir: string | undefined; @@ -128,12 +136,12 @@ export async function runServerConformanceTest( // connection rather than the stateful latest-spec default. Extension // scenarios are off-timeline; today every extension in this repo lives // on draft, so they fall under the same inference. - const resolvedSpecVersion = - specVersion ?? - ('extensionId' in scenario.source || + const defaultSpecVersion = + 'extensionId' in scenario.source || scenario.source.introducedIn === DRAFT_PROTOCOL_VERSION ? DRAFT_PROTOCOL_VERSION - : LATEST_SPEC_VERSION); + : LATEST_SPEC_VERSION; + const resolvedSpecVersion = specVersion ?? defaultSpecVersion; console.log( `Running client scenario '${scenarioName}' against server: ${serverUrl}` @@ -152,11 +160,21 @@ export async function runServerConformanceTest( ); checks.push(...wireSchemaChecks(resolvedSpecVersion)); + const coverage = scenarioCoverage( + 'server', + scenarioName, + checks, + resolvedSpecVersion === defaultSpecVersion + ); + if (resultDir) { await fs.writeFile( path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2) ); + if (coverage) { + await writeCoverage(resultDir, 'server', scenarioName, coverage); + } console.log(`Results saved to ${resultDir}`); } @@ -164,14 +182,16 @@ export async function runServerConformanceTest( return { checks, resultDir, - scenarioDescription: scenario.description + scenarioDescription: scenario.description, + coverage }; } export function printServerResults( checks: ConformanceCheck[], scenarioDescription: string, - verbose: boolean = false + verbose: boolean = false, + coverage?: Coverage | null ): { passed: number; failed: number; @@ -195,6 +215,7 @@ export function printServerResults( console.log( `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` ); + if (coverage) console.log(formatCoverage(coverage)); if (failed > 0) { console.log('\n=== Failed Checks ==='); @@ -213,7 +234,11 @@ export function printServerResults( } export function printServerSummary( - allResults: { scenario: string; checks: ConformanceCheck[] }[] + allResults: { + scenario: string; + checks: ConformanceCheck[]; + coverage?: Coverage | null; + }[] ): { totalPassed: number; totalFailed: number } { console.log('\n\n=== SUMMARY ==='); let totalPassed = 0; @@ -227,7 +252,8 @@ export function printServerSummary( const status = failed === 0 ? '✓' : '✗'; console.log( - `${status} ${result.scenario}: ${passed} passed, ${failed} failed` + `${status} ${result.scenario}: ${passed} passed, ${failed} failed` + + coverageSuffix(result.coverage) ); } diff --git a/src/traceability/index.test.ts b/src/traceability/index.test.ts index 721365e6..f0f5c0b1 100644 --- a/src/traceability/index.test.ts +++ b/src/traceability/index.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect } from 'vitest'; -import { computeTraceability, DeclaredSep } from './index'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs'; +import os from 'os'; +import path from 'path'; +import { + collectScenarioBaselines, + computeTraceability, + DeclaredSep, + scenarioKeyForResultDir +} from './index'; const decl = ( sep: number, @@ -122,3 +130,67 @@ describe('computeTraceability', () => { expect(m.source).toBeNull(); }); }); + +describe('scenario baselines', () => { + const client = new Set(['initialize', 'auth/dpop', 'server-ish']); + const server = new Set(['tools-list', 'server-stateless']); + + it('maps result dirs to scenario keys', () => { + const ts = '-2026-11-13T20-00-00-000Z'; + expect( + scenarioKeyForResultDir(`server-tools-list${ts}`, client, server) + ).toBe('server:tools-list'); + expect( + scenarioKeyForResultDir(`server-server-stateless${ts}`, client, server) + ).toBe('server:server-stateless'); + expect( + scenarioKeyForResultDir(path.join('auth', `dpop${ts}`), client, server) + ).toBe('client:auth/dpop'); + // A client scenario whose name starts with server- is not a server run. + expect(scenarioKeyForResultDir(`server-ish${ts}`, client, server)).toBe( + 'client:server-ish' + ); + expect( + scenarioKeyForResultDir(`authorization-server-x${ts}`, client, server) + ).toBeNull(); + }); + + it('collects counted check IDs per scenario from a results dir', () => { + const dir = mkdtempSync(path.join(os.tmpdir(), 'trace-')); + try { + const write = (rel: string, checks: object[]) => { + mkdirSync(path.join(dir, rel), { recursive: true }); + writeFileSync( + path.join(dir, rel, 'checks.json'), + JSON.stringify(checks) + ); + }; + write('server-tools-list-2026-11-13T20-00-00-000Z', [ + { id: 'tools-list', status: 'SUCCESS' }, + { id: 'incoming-request', status: 'INFO' }, + { id: 'wire-schema-valid', status: 'SUCCESS' } + ]); + write(path.join('auth', 'dpop-2026-11-13T20-00-00-000Z'), [ + { id: 'sep-1932-proof', status: 'FAILURE' }, + { id: 'sep-1932-skip', status: 'SKIPPED' } + ]); + const got = collectScenarioBaselines(dir, client, server); + const m = computeTraceability({ + declared: [], + emitted: new Set(), + scenarios: got + }); + expect(m.scenarios).toEqual({ + 'client:auth/dpop': { expected: ['sep-1932-proof'] }, + 'server:tools-list': { expected: ['tools-list', 'wire-schema-valid'] } + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('omits scenarios when none were collected', () => { + const m = computeTraceability({ declared: [], emitted: new Set() }); + expect(m.scenarios).toBeUndefined(); + }); +}); diff --git a/src/traceability/index.ts b/src/traceability/index.ts index 055076da..8afe5c90 100644 --- a/src/traceability/index.ts +++ b/src/traceability/index.ts @@ -2,11 +2,13 @@ import { Command } from 'commander'; import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs'; import path from 'path'; import { parse as parseYaml } from 'yaml'; +import { listClientScenarios, listScenarios } from '../scenarios'; import { TRACEABILITY_SCHEMA_VERSION, TraceabilityManifest, ExcludedRequirement, RequirementTraceability, + ScenarioBaseline, SepTraceability, UnkeyedRequirement } from './types'; @@ -54,6 +56,7 @@ export function computeTraceability(args: { declared: DeclaredSep[]; emitted: Set; source?: string | null; + scenarios?: Record; }): TraceabilityManifest { const { declared, emitted } = args; @@ -132,10 +135,51 @@ export function computeTraceability(args: { schemaVersion: TRACEABILITY_SCHEMA_VERSION, docs: DOCS, source: args.source ?? null, - seps + seps, + ...(args.scenarios && Object.keys(args.scenarios).length + ? { scenarios: sortedScenarios(args.scenarios) } + : {}) }; } +function sortedScenarios( + scenarios: Record +): Record { + const out: Record = {}; + for (const key of Object.keys(scenarios).sort()) { + out[key] = { expected: [...new Set(scenarios[key].expected)].sort() }; + } + return out; +} + +const RESULT_DIR_TIMESTAMP = /-\d{4}-\d{2}-\d{2}T[\d-]+Z$/; +const COUNTED_STATUS = new Set(['SUCCESS', 'FAILURE', 'WARNING']); + +/** + * Pure: map a results-relative directory (from createResultDir) to its + * scenario key. Server runs are prefixed `server-`. Client runs carry the bare + * scenario name, which may contain a slash. The registered scenario lists + * settle the ambiguous case where a client scenario name starts with + * `server-`. Returns null for directories that are neither, such as + * authorization-server runs. + */ +export function scenarioKeyForResultDir( + relDir: string, + clientScenarios: ReadonlySet, + serverScenarios: ReadonlySet +): string | null { + const name = relDir + .split(path.sep) + .join('/') + .replace(RESULT_DIR_TIMESTAMP, ''); + if (name.startsWith('server-')) { + const rest = name.slice('server-'.length); + if (serverScenarios.has(rest)) return `server:${rest}`; + } + if (clientScenarios.has(name)) return `client:${name}`; + return null; +} + /** Serialize deterministically (sorted SEP keys, trailing newline). */ export function serializeManifest(manifest: TraceabilityManifest): string { return JSON.stringify(manifest, null, 2) + '\n'; @@ -168,6 +212,46 @@ export function collectEmittedIds(resultsDir: string): Set { return ids; } +/** + * Collect each scenario's counted check IDs from a results dir. Every check + * ID counts here, not only sep-NNNN-* ones, since a thin run can drop any of + * them. + */ +export function collectScenarioBaselines( + resultsDir: string, + clientScenarios: ReadonlySet, + serverScenarios: ReadonlySet +): Record { + const out: Record = {}; + let entries: string[]; + try { + entries = readdirSync(resultsDir, { recursive: true, encoding: 'utf-8' }); + } catch { + return out; + } + for (const rel of entries) { + if (path.basename(rel) !== 'checks.json') continue; + const key = scenarioKeyForResultDir( + path.dirname(rel), + clientScenarios, + serverScenarios + ); + if (!key) continue; + try { + const arr = JSON.parse(readFileSync(path.join(resultsDir, rel), 'utf8')); + if (!Array.isArray(arr)) continue; + const ids = (out[key] ??= { expected: [] }).expected; + for (const c of arr) { + if (c && typeof c.id === 'string' && COUNTED_STATUS.has(c.status)) + ids.push(c.id); + } + } catch { + // skip unreadable/partial result files + } + } + return out; +} + export function gatherDeclared(sepsDir = SEPS_DIR): DeclaredSep[] { const out: DeclaredSep[] = []; const files = readdirSync(sepsDir) @@ -276,10 +360,17 @@ export function createTraceabilityCommand(): Command { process.exit(1); } + const scenarios = collectScenarioBaselines( + options.results, + new Set(listScenarios()), + new Set(listClientScenarios()) + ); + const manifest = computeTraceability({ declared, emitted, - source: options.source ?? null + source: options.source ?? null, + scenarios }); const serialized = serializeManifest(manifest); const untestedTotal = Object.values(manifest.seps).reduce( diff --git a/src/traceability/types.ts b/src/traceability/types.ts index 8edc0221..bfb962b6 100644 --- a/src/traceability/types.ts +++ b/src/traceability/types.ts @@ -5,10 +5,10 @@ * exists* for each declared SEP requirement. It says NOTHING about whether any * particular SDK passes that scenario — per-SDK pass/fail lives in `tier-check`. * - * Joining the two is a future goal, NOT possible today: tier-check reports at - * scenario granularity and does not currently expose per-check IDs, while this - * manifest carries check IDs but not scenario names. Wiring plan.mcp.io's two - * feeds together needs one side to add the missing column first. + * Joining the two is a future goal. tier-check reports at scenario + * granularity and does not expose per-check IDs. This manifest's `scenarios` + * map now carries scenario names next to check IDs, which is one half of that + * join. */ export const TRACEABILITY_SCHEMA_VERSION = 1; @@ -64,6 +64,15 @@ export interface SepTraceability { }; } +/** + * The check IDs one scenario emitted in the reference run, with status + * SUCCESS, FAILURE or WARNING. A later run compares against this to show + * checks that never fired (see src/coverage.ts). + */ +export interface ScenarioBaseline { + expected: string[]; +} + export interface TraceabilityManifest { schemaVersion: number; /** Pointer to where this file's semantics are documented (not prose-in-data). */ @@ -76,4 +85,9 @@ export interface TraceabilityManifest { source: string | null; /** Keyed by SEP number (as a string). */ seps: Record; + /** + * Keyed by `client:` or `server:`, where the prefix is + * the CLI mode that ran it. Absent in manifests built before #451. + */ + scenarios?: Record; }