Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions src/coverage.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import { describe, expect, it } from 'vitest';
import {
computeCoverage,
coverageSuffix,
expectedFor,
formatCoverage
} from './coverage';
import type { ConformanceCheck, CheckStatus } from './types';
import type { TraceabilityManifest } from './traceability/types';

function check(id: string, status: CheckStatus = 'SUCCESS'): ConformanceCheck {
return {
id,
name: id,
description: id,
status,
timestamp: '2026-01-01T00:00:00.000Z'
};
}

const EXPECTED = ['a', 'b', 'c', 'd', 'wire-schema-valid'];

describe('computeCoverage', () => {
it('reports every expected check when the run is complete', () => {
const c = computeCoverage(
EXPECTED,
EXPECTED.map((id) => check(id))
);
expect(c).toEqual({ expected: 5, emitted: 5, missing: [] });
expect(coverageSuffix(c)).toBe('');
});

it('shows a thin run that passes every check it emitted', () => {
// Every emitted check is green, so pass/fail reads as a clean pass.
const checks = [check('b'), check('wire-schema-valid')];
const c = computeCoverage(EXPECTED, checks)!;
expect(checks.every((x) => x.status === 'SUCCESS')).toBe(true);
expect(c).toEqual({ expected: 5, emitted: 2, missing: ['a', 'c', 'd'] });
expect(coverageSuffix(c)).toBe(' (coverage 2/5)');
});

it('shows how much of the scenario a setup failure skipped', () => {
const c = computeCoverage(EXPECTED, [check('a', 'FAILURE')])!;
expect(c.emitted).toBe(1);
expect(c.missing).toEqual(['b', 'c', 'd', 'wire-schema-valid']);
});

it('does not count INFO or SKIPPED rows as emitted', () => {
const c = computeCoverage(
['a', 'b'],
[check('a', 'INFO'), check('b', 'SKIPPED')]
)!;
expect(c.emitted).toBe(0);
});

it('counts WARNING as emitted', () => {
expect(computeCoverage(['a'], [check('a', 'WARNING')])!.missing).toEqual(
[]
);
});

it('returns null without a baseline', () => {
expect(computeCoverage(undefined, [check('a')])).toBeNull();
expect(computeCoverage([], [check('a')])).toBeNull();
});
});

describe('expectedFor', () => {
const manifest: TraceabilityManifest = {
schemaVersion: 1,
docs: '',
source: 'typescript-sdk@test',
seps: {},
scenarios: {
'server:tools-list': { expected: ['x'] },
'client:auth/dpop': { expected: ['y'] }
}
};

it('keys by CLI mode and scenario name', () => {
expect(expectedFor('server', 'tools-list', true, manifest)).toEqual(['x']);
expect(expectedFor('client', 'auth/dpop', true, manifest)).toEqual(['y']);
expect(expectedFor('client', 'tools-list', true, manifest)).toBeUndefined();
});

it('gives no baseline away from the default spec version', () => {
expect(
expectedFor('server', 'tools-list', false, manifest)
).toBeUndefined();
});

it('gives no baseline for a manifest built before scenarios existed', () => {
const old = { ...manifest, scenarios: undefined };
expect(expectedFor('server', 'tools-list', true, old)).toBeUndefined();
});
});

describe('formatCoverage', () => {
it('lists missing IDs and caps the list', () => {
const missing = Array.from({ length: 12 }, (_, i) => `m${i}`);
const line = formatCoverage({ expected: 15, emitted: 3, missing });
expect(line).toMatch(/^Coverage: 3\/15 expected checks emitted\. /);
expect(line).toContain('m9');
expect(line).not.toContain('m10');
expect(line).toMatch(/and 2 more$/);
});
});
122 changes: 122 additions & 0 deletions src/coverage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/**
* Expected-vs-emitted check coverage per scenario (issue #451).
*
* A scenario can pass while emitting only a fraction of its checks, for
* example when the implementation under test fails early and the checks that
* would have run never fire. Pass/fail cannot show that. This module compares
* the check IDs a run emitted against the IDs the same scenario emitted when
* the traceability workflow ran it against the reference SDK, and reports
* the ones that are missing.
*
* The baseline lives in src/seps/traceability.json under `scenarios`, so SDKs
* do not keep their own per-scenario snapshots. Coverage is advisory: it is
* printed and written to coverage.json, and it never changes the exit code.
*/
import { promises as fs } from 'fs';
import path from 'path';
import type { ConformanceCheck } from './types';
import type { TraceabilityManifest } from './traceability/types';
import manifestJson from './seps/traceability.json';

/** Which CLI mode ran the scenario: `client` tests a client, `server` a server. */
export type Side = 'client' | 'server';

/** Statuses that mean the check actually ran. INFO and SKIPPED do not count. */
const COUNTED = new Set(['SUCCESS', 'FAILURE', 'WARNING']);

export interface Coverage {
/** Check IDs the reference SDK emitted for this scenario. */
expected: number;
/** How many of those this run emitted. */
emitted: number;
/** Expected check IDs this run did not emit, sorted. */
missing: string[];
}

export function scenarioKey(side: Side, scenario: string): string {
return `${side}:${scenario}`;
}

export function countedIds(checks: ConformanceCheck[]): Set<string> {
return new Set(checks.filter((c) => COUNTED.has(c.status)).map((c) => c.id));
}

/** Pure: null when there is no baseline, or the baseline is empty. */
export function computeCoverage(
expected: string[] | undefined,
checks: ConformanceCheck[]
): Coverage | null {
if (!expected || expected.length === 0) return null;
const emitted = countedIds(checks);
const missing = expected.filter((id) => !emitted.has(id)).sort();
return {
expected: expected.length,
emitted: expected.length - missing.length,
missing
};
}

const bundled = manifestJson as unknown as TraceabilityManifest;

/**
* The reference run uses each scenario's default spec version. A run at any
* other version can legitimately emit a different set, so it gets no baseline
* rather than a false gap.
*/
export function expectedFor(
side: Side,
scenario: string,
atDefaultSpecVersion: boolean,
manifest: TraceabilityManifest = bundled
): string[] | undefined {
if (!atDefaultSpecVersion) return undefined;
return manifest.scenarios?.[scenarioKey(side, scenario)]?.expected;
}

export function scenarioCoverage(
side: Side,
scenario: string,
checks: ConformanceCheck[],
atDefaultSpecVersion: boolean
): Coverage | null {
return computeCoverage(
expectedFor(side, scenario, atDefaultSpecVersion),
checks
);
}

const MAX_LISTED = 10;

/** One line for the per-scenario results, plus the missing IDs if any. */
export function formatCoverage(c: Coverage): string {
const head = `Coverage: ${c.emitted}/${c.expected} expected checks emitted`;
if (c.missing.length === 0) return head;
const listed = c.missing.slice(0, MAX_LISTED).join(', ');
const more =
c.missing.length > MAX_LISTED
? `, and ${c.missing.length - MAX_LISTED} more`
: '';
return `${head}. Not emitted: ${listed}${more}`;
}

/** Suffix for a summary line. Empty when coverage is complete or unknown. */
export function coverageSuffix(c: Coverage | null | undefined): string {
if (!c || c.missing.length === 0) return '';
return ` (coverage ${c.emitted}/${c.expected})`;
}

export async function writeCoverage(
resultDir: string,
side: Side,
scenario: string,
c: Coverage
): Promise<void> {
await fs.writeFile(
path.join(resultDir, 'coverage.json'),
JSON.stringify(
{ scenario, side, baseline: bundled.source, ...c },
null,
2
) + '\n'
);
}
25 changes: 19 additions & 6 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env node

import { Command } from 'commander';
import { Coverage, coverageSuffix } from './coverage';
import { ZodError } from 'zod';
import { promises as fs } from 'fs';
import {
Expand Down Expand Up @@ -337,6 +338,7 @@ program
scenario: scenarioName,
checks: result.checks,
skipped: result.skipped,
coverage: result.coverage,
error: null
};
} catch (error) {
Expand All @@ -354,6 +356,7 @@ program
}
],
skipped: undefined,
coverage: undefined,
error
};
}
Expand Down Expand Up @@ -393,7 +396,8 @@ program
const status = failed === 0 && warnings === 0 ? '✓' : '✗';
const warningStr = warnings > 0 ? `, ${warnings} warnings` : '';
console.log(
`${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}`
`${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` +
coverageSuffix(result.coverage)
);

if (verbose && failed > 0) {
Expand Down Expand Up @@ -492,7 +496,8 @@ program
result.checks,
verbose,
result.clientOutput,
result.allowClientError
result.allowClientError,
result.coverage
);

if (options.expectedFailures) {
Expand Down Expand Up @@ -603,7 +608,8 @@ program
const { failed } = printServerResults(
result.checks,
result.scenarioDescription,
verbose
verbose,
result.coverage
);

if (options.expectedFailures) {
Expand Down Expand Up @@ -662,8 +668,11 @@ program
`Running ${selection} (${scenarios.length} scenarios) against ${validated.url}\n`
);

const allResults: { scenario: string; checks: ConformanceCheck[] }[] =
[];
const allResults: {
scenario: string;
checks: ConformanceCheck[];
coverage?: Coverage | null;
}[] = [];

for (const scenarioName of scenarios) {
console.log(`\n=== Running scenario: ${scenarioName} ===`);
Expand All @@ -682,7 +691,11 @@ program
timeout
)
);
allResults.push({ scenario: scenarioName, checks: result.checks });
allResults.push({
scenario: scenarioName,
checks: result.checks,
coverage: result.coverage
});
} catch (error) {
console.error(`Failed to run scenario ${scenarioName}:`, error);
allResults.push({
Expand Down
24 changes: 22 additions & 2 deletions src/runner/client.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import { spawn } from 'child_process';
import {
Coverage,
formatCoverage,
scenarioCoverage,
writeCoverage
} from '../coverage';
import { promises as fs } from 'fs';
import path from 'path';
import {
Expand Down Expand Up @@ -159,6 +165,7 @@ export async function runConformanceTest(
resultDir?: string;
allowClientError?: boolean;
skipped?: boolean;
coverage?: Coverage | null;
}> {
let resultDir: string | undefined;

Expand Down Expand Up @@ -222,11 +229,21 @@ export async function runConformanceTest(
const checks = scenario.getChecks();
checks.push(...wireSchemaChecks(resolvedVersion));

const coverage = scenarioCoverage(
'client',
scenarioName,
checks,
resolvedVersion === resolveScenarioSpecVersion(scenario.source, undefined)
);

if (resultDir) {
await fs.writeFile(
path.join(resultDir, 'checks.json'),
JSON.stringify(checks, null, 2)
);
if (coverage) {
await writeCoverage(resultDir, 'client', scenarioName, coverage);
}

await fs.writeFile(
path.join(resultDir, 'stdout.txt'),
Expand All @@ -245,7 +262,8 @@ export async function runConformanceTest(
checks,
clientOutput,
resultDir,
allowClientError: scenario.allowClientError
allowClientError: scenario.allowClientError,
coverage
};
} finally {
await scenario.stop();
Expand All @@ -256,7 +274,8 @@ export function printClientResults(
checks: ConformanceCheck[],
verbose: boolean = false,
clientOutput?: ClientExecutionResult,
allowClientError: boolean = false
allowClientError: boolean = false,
coverage?: Coverage | null
): {
passed: number;
failed: number;
Expand Down Expand Up @@ -295,6 +314,7 @@ export function printClientResults(
console.error(
`Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings`
);
if (coverage) console.error(formatCoverage(coverage));

if (clientTimedOut) {
console.error(`\n⚠️ CLIENT TIMED OUT - Test incomplete`);
Expand Down
Loading
Loading