Skip to content

Commit 05104f7

Browse files
committed
feat: support "draft" as a first-class spec version target
- Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253
1 parent 90b2334 commit 05104f7

9 files changed

Lines changed: 116 additions & 46 deletions

File tree

README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
6464
- `--command` - The command to run your MCP client (can include flags)
6565
- `--scenario` - The test scenario to run (e.g., "initialize")
6666
- `--suite` - Run a suite of tests in parallel (e.g., "auth")
67+
- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `draft`). `draft` runs the latest dated release plus any draft-only scenarios
6768
- `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
6869
- `--timeout` - Timeout in milliseconds (default: 30000)
6970
- `--verbose` - Show verbose output
7071

71-
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
72+
The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its value is also forwarded as `MCP_CONFORMANCE_SPEC_VERSION`; example clients that target multiple spec versions can read this to select a `protocolVersion` at runtime. SDKs that hard-code their protocol version can ignore it — the harness accepts any draft-prefixed `protocolVersion` (e.g., `DRAFT-2026-v1`) regardless.
7273

7374
### Server Testing
7475

src/checks/checks.test.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,23 @@ describe('createClientInitializationCheck', () => {
6868
expect(check.errorMessage).toContain('Client version missing');
6969
});
7070

71+
it.each(['DRAFT-2026-v1', 'draft'])(
72+
'should accept draft protocol version %s',
73+
(protocolVersion) => {
74+
const request = {
75+
protocolVersion,
76+
clientInfo: {
77+
name: 'TestClient',
78+
version: '1.0.0'
79+
}
80+
};
81+
82+
const check = createClientInitializationCheck(request);
83+
expect(check.status).toBe('SUCCESS');
84+
expect(check.errorMessage).toBeUndefined();
85+
}
86+
);
87+
7188
it('should support custom expected spec version', () => {
7289
const request = {
7390
protocolVersion: '2024-11-05',

src/checks/client.ts

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ConformanceCheck, CheckStatus } from '../types';
1+
import { ConformanceCheck, CheckStatus, LATEST_SPEC_VERSION } from '../types';
22

33
export function createServerInfoCheck(serverInfo: {
44
name: string;
@@ -23,20 +23,30 @@ export function createServerInfoCheck(serverInfo: {
2323
};
2424
}
2525

26-
// Valid MCP protocol versions
27-
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];
26+
// Dated protocol versions the mock server will accept on initialize.
27+
const VALID_PROTOCOL_VERSIONS = ['2025-06-18', LATEST_SPEC_VERSION];
28+
29+
// The spec repo's schema/draft/schema.ts uses a moving identifier of the form
30+
// DRAFT-YYYY-vN (e.g. "DRAFT-2026-v1"). Accept any draft-prefixed value so SEP
31+
// authors can run the suite against an SDK tracking the in-progress spec
32+
// without pinning the exact draft revision here.
33+
export function isDraftProtocolVersion(version: unknown): boolean {
34+
return typeof version === 'string' && /^draft/i.test(version);
35+
}
2836

2937
export function createClientInitializationCheck(
3038
initializeRequest: any,
31-
expectedSpecVersion: string = '2025-11-25'
39+
expectedSpecVersion: string = LATEST_SPEC_VERSION
3240
): ConformanceCheck {
3341
const protocolVersionSent = initializeRequest?.protocolVersion;
3442

3543
// Accept known valid versions OR custom expected version (for backward compatibility)
3644
const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion)
3745
? VALID_PROTOCOL_VERSIONS
3846
: [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion];
39-
const versionMatch = validVersions.includes(protocolVersionSent);
47+
const versionMatch =
48+
validVersions.includes(protocolVersionSent) ||
49+
isDraftProtocolVersion(protocolVersionSent);
4050

4151
const errors: string[] = [];
4252
if (!protocolVersionSent) errors.push('Protocol version not provided');

src/index.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,8 @@ program
152152
options.command,
153153
scenarioName,
154154
timeout,
155-
outputDir
155+
outputDir,
156+
specVersionFilter
156157
);
157158
return {
158159
scenario: scenarioName,
@@ -259,7 +260,8 @@ program
259260
validated.command,
260261
validated.scenario,
261262
timeout,
262-
outputDir
263+
outputDir,
264+
specVersionFilter
263265
);
264266

265267
const { overallFailure } = printClientResults(

src/runner/client.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { spawn } from 'child_process';
22
import { promises as fs } from 'fs';
33
import path from 'path';
4-
import { ConformanceCheck } from '../types';
4+
import { ConformanceCheck, SpecVersion } from '../types';
55
import { getScenario } from '../scenarios';
66
import { createResultDir, formatPrettyChecks } from './utils';
77

@@ -17,7 +17,8 @@ async function executeClient(
1717
scenarioName: string,
1818
serverUrl: string,
1919
timeout: number = 30000,
20-
context?: Record<string, unknown>
20+
context?: Record<string, unknown>,
21+
specVersion?: SpecVersion
2122
): Promise<ClientExecutionResult> {
2223
const commandParts = command.split(' ');
2324
const executable = commandParts[0];
@@ -34,6 +35,9 @@ async function executeClient(
3435
// 3. Semantic separation: scenario identifies "which test", context provides "test data"
3536
const env = { ...process.env };
3637
env.MCP_CONFORMANCE_SCENARIO = scenarioName;
38+
if (specVersion) {
39+
env.MCP_CONFORMANCE_SPEC_VERSION = specVersion;
40+
}
3741
if (context) {
3842
// Include scenario name in context for discriminated union parsing
3943
env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
@@ -92,7 +96,8 @@ export async function runConformanceTest(
9296
clientCommand: string,
9397
scenarioName: string,
9498
timeout: number = 30000,
95-
outputDir?: string
99+
outputDir?: string,
100+
specVersion?: SpecVersion
96101
): Promise<{
97102
checks: ConformanceCheck[];
98103
clientOutput: ClientExecutionResult;
@@ -123,7 +128,8 @@ export async function runConformanceTest(
123128
scenarioName,
124129
urls.serverUrl,
125130
timeout,
126-
urls.context
131+
urls.context,
132+
specVersion
127133
);
128134

129135
// Print stdout/stderr if client exited with nonzero code

src/scenarios/client/initialize.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ import {
33
Scenario,
44
ScenarioUrls,
55
ConformanceCheck,
6-
SpecVersion
6+
SpecVersion,
7+
LATEST_SPEC_VERSION
78
} from '../../types';
89
import { clientChecks } from '../../checks/index';
10+
import { isDraftProtocolVersion } from '../../checks/client';
911

1012
export class InitializeScenario implements Scenario {
1113
name = 'initialize';
@@ -117,11 +119,13 @@ export class InitializeScenario implements Scenario {
117119
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
118120

119121
// Echo back client's version if valid, otherwise use latest
120-
const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
122+
const VALID_VERSIONS = ['2025-06-18', LATEST_SPEC_VERSION];
121123
const clientVersion = initializeRequest?.protocolVersion;
122-
const responseVersion = VALID_VERSIONS.includes(clientVersion)
123-
? clientVersion
124-
: '2025-11-25';
124+
const responseVersion =
125+
VALID_VERSIONS.includes(clientVersion) ||
126+
isDraftProtocolVersion(clientVersion)
127+
? clientVersion
128+
: LATEST_SPEC_VERSION;
125129

126130
const response = {
127131
jsonrpc: '2.0',

src/scenarios/index.ts

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ import {
22
Scenario,
33
ClientScenario,
44
ClientScenarioForAuthorizationServer,
5-
SpecVersion
5+
SpecVersion,
6+
DATED_SPEC_VERSIONS,
7+
LATEST_SPEC_VERSION
68
} from '../types';
79
import { InitializeScenario } from './client/initialize';
810
import { ToolsCallScenario } from './client/tools_call';
@@ -257,9 +259,7 @@ export { listMetadataScenarios };
257259

258260
// All valid spec versions, used by the CLI to validate --spec-version input.
259261
export const ALL_SPEC_VERSIONS: SpecVersion[] = [
260-
'2025-03-26',
261-
'2025-06-18',
262-
'2025-11-25',
262+
...DATED_SPEC_VERSIONS,
263263
'draft',
264264
'extension'
265265
];
@@ -273,23 +273,39 @@ export function resolveSpecVersion(value: string): SpecVersion {
273273
process.exit(1);
274274
}
275275

276+
// `draft` selects everything in the latest dated release plus scenarios tagged
277+
// draft-only, so SEP authors can run the full suite against an SDK tracking the
278+
// in-progress spec without retagging core scenarios.
279+
function matchesSpecVersion(
280+
scenario: { specVersions: SpecVersion[] },
281+
version: SpecVersion
282+
): boolean {
283+
if (version === 'draft') {
284+
return (
285+
scenario.specVersions.includes('draft') ||
286+
scenario.specVersions.includes(LATEST_SPEC_VERSION)
287+
);
288+
}
289+
return scenario.specVersions.includes(version);
290+
}
291+
276292
export function listScenariosForSpec(version: SpecVersion): string[] {
277293
return scenariosList
278-
.filter((s) => s.specVersions.includes(version))
294+
.filter((s) => matchesSpecVersion(s, version))
279295
.map((s) => s.name);
280296
}
281297

282298
export function listClientScenariosForSpec(version: SpecVersion): string[] {
283299
return allClientScenariosList
284-
.filter((s) => s.specVersions.includes(version))
300+
.filter((s) => matchesSpecVersion(s, version))
285301
.map((s) => s.name);
286302
}
287303

288304
export function listClientScenariosForAuthorizationServerForSpec(
289305
version: SpecVersion
290306
): string[] {
291307
return allClientScenariosListForAuthorizationServer
292-
.filter((s) => s.specVersions.includes(version))
308+
.filter((s) => matchesSpecVersion(s, version))
293309
.map((s) => s.name);
294310
}
295311

src/scenarios/spec-version.test.ts

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ import {
33
listScenarios,
44
listClientScenarios,
55
listScenariosForSpec,
6+
listDraftScenarios,
67
getScenarioSpecVersions,
78
ALL_SPEC_VERSIONS
89
} from './index';
10+
import { DATED_SPEC_VERSIONS, LATEST_SPEC_VERSION } from '../types';
911

1012
describe('specVersions helpers', () => {
1113
it('every Scenario has specVersions', () => {
@@ -69,26 +71,33 @@ describe('specVersions helpers', () => {
6971
}
7072
});
7173

72-
it('draft and extension scenarios are isolated', () => {
73-
const draft = listScenariosForSpec('draft');
74-
for (const name of draft) {
75-
expect(getScenarioSpecVersions(name)).toContain('draft');
74+
it('--spec-version draft is a superset of the latest dated release', () => {
75+
const latest = new Set(listScenariosForSpec(LATEST_SPEC_VERSION));
76+
const draft = new Set(listScenariosForSpec('draft'));
77+
for (const name of latest) {
78+
expect(draft.has(name)).toBe(true);
7679
}
77-
const ext = listScenariosForSpec('extension');
78-
for (const name of ext) {
79-
expect(getScenarioSpecVersions(name)).toContain('extension');
80+
for (const name of listDraftScenarios()) {
81+
expect(draft.has(name)).toBe(true);
8082
}
8183
});
8284

83-
it('draft scenarios are not in dated versions', () => {
84-
const draft = listScenariosForSpec('draft');
85-
const dated = new Set([
86-
...listScenariosForSpec('2025-03-26'),
87-
...listScenariosForSpec('2025-06-18'),
88-
...listScenariosForSpec('2025-11-25')
89-
]);
90-
for (const name of draft) {
91-
expect(dated.has(name)).toBe(false);
85+
it('draft-tagged scenarios are not also tagged with a dated version', () => {
86+
for (const name of listDraftScenarios()) {
87+
const versions = getScenarioSpecVersions(name)!;
88+
for (const dated of DATED_SPEC_VERSIONS) {
89+
expect(
90+
versions,
91+
`scenario "${name}" is tagged with both 'draft' and '${dated}'`
92+
).not.toContain(dated);
93+
}
94+
}
95+
});
96+
97+
it('extension scenarios are isolated', () => {
98+
const ext = listScenariosForSpec('extension');
99+
for (const name of ext) {
100+
expect(getScenarioSpecVersions(name)).toContain('extension');
92101
}
93102
});
94103
});

src/types.ts

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,17 @@ export interface ConformanceCheck {
2323
logs?: string[];
2424
}
2525

26-
export type SpecVersion =
27-
| '2025-03-26'
28-
| '2025-06-18'
29-
| '2025-11-25'
30-
| 'draft'
31-
| 'extension';
26+
export const DATED_SPEC_VERSIONS = [
27+
'2025-03-26',
28+
'2025-06-18',
29+
'2025-11-25'
30+
] as const;
31+
32+
export type DatedSpecVersion = (typeof DATED_SPEC_VERSIONS)[number];
33+
34+
export const LATEST_SPEC_VERSION: DatedSpecVersion = '2025-11-25';
35+
36+
export type SpecVersion = DatedSpecVersion | 'draft' | 'extension';
3237

3338
export interface ScenarioUrls {
3439
serverUrl: string;

0 commit comments

Comments
 (0)