feat: support "draft" as a first-class spec version target (#255)

pcarleton · web-flow · commit 17f1f93cc070 · 2026-04-30T11:37:32.000+01:00
* feat: support "draft" as a first-class spec version target - Add LATEST_SPEC_VERSION and DATED_SPEC_VERSIONS constants so the next spec release is a one-line change in types.ts. - Accept "draft" as a valid protocolVersion in the initialize check and mock-server response. - --spec-version draft now selects latest-dated scenarios plus draft-tagged ones, so SEP authors can run the full suite against an SDK tracking the in-progress spec without retagging core scenarios. - Forward --spec-version to the client process via MCP_CONFORMANCE_SPEC_VERSION so SDK examples can pick the matching protocolVersion. Closes #253 * review: collapse SpecVersion to wire strings; dedup NEGOTIABLE_PROTOCOL_VERSIONS; split out 'extension' Addresses review feedback from @felixweinberger and @mikekistler: - SpecVersion is now always a wire protocolVersion string (DatedSpecVersion | typeof DRAFT_PROTOCOL_VERSION). The separate 'draft' tag literal is gone from the type system; 'draft' survives only as a CLI input alias in resolveSpecVersion. Removes the tag-vs-wire confusion and the specVersionToProtocolVersion mapping. - NEGOTIABLE_PROTOCOL_VERSIONS in types.ts is the single source for what the mock server accepts on initialize (was duplicated in checks/client.ts and scenarios/client/initialize.ts). - 'extension' moved out of SpecVersion into a separate ScenarioSpecTag type. --spec-version extension is no longer valid; extension scenarios remain reachable via --suite extensions. - Draft-tagged scenarios now use the DRAFT_PROTOCOL_VERSION constant so a draft revision bump is a one-line change in types.ts.
diff --git a/README.md b/README.md
@@ -64,11 +64,12 @@ npx @modelcontextprotocol/conformance client --command "<client-command>" --scen
 - `--command` - The command to run your MCP client (can include flags)
 - `--scenario` - The test scenario to run (e.g., "initialize")
 - `--suite` - Run a suite of tests in parallel (e.g., "auth")
+- `--spec-version <version>` - Filter scenarios by spec version (e.g., `2025-11-25`, `DRAFT-2026-v1`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios
 - `--expected-failures <path>` - Path to YAML baseline file of known failures (see [Expected Failures](#expected-failures))
 - `--timeout` - Timeout in milliseconds (default: 30000)
 - `--verbose` - Show verbose output
 
-The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data.
+The framework appends `<server-url>` as an argument to your command and sets the `MCP_CONFORMANCE_SCENARIO` environment variable to the scenario name. For scenarios that require additional context (e.g., client credentials), the `MCP_CONFORMANCE_CONTEXT` environment variable contains a JSON object with scenario-specific data. When `--spec-version` is passed, its resolved value is forwarded to the client process as `MCP_CONFORMANCE_PROTOCOL_VERSION`; example clients can use this value directly as their `protocolVersion`. SDKs that hard-code their protocol version can ignore it.
 
 ### Server Testing
 
diff --git a/src/checks/checks.test.ts b/src/checks/checks.test.ts
@@ -1,4 +1,5 @@
 import { createClientInitializationCheck } from './client';
+import { DRAFT_PROTOCOL_VERSION } from '../types';
 
 describe('createClientInitializationCheck', () => {
   it('should return SUCCESS for a valid initialize request', () => {
@@ -68,6 +69,31 @@ describe('createClientInitializationCheck', () => {
     expect(check.errorMessage).toContain('Client version missing');
   });
 
+  it('should accept the current draft protocol version', () => {
+    const request = {
+      protocolVersion: DRAFT_PROTOCOL_VERSION,
+      clientInfo: { name: 'TestClient', version: '1.0.0' }
+    };
+
+    const check = createClientInitializationCheck(request);
+    expect(check.status).toBe('SUCCESS');
+    expect(check.errorMessage).toBeUndefined();
+  });
+
+  it.each(['DRAFT-2025-v1', 'draft'])(
+    'should reject stale or non-canonical draft version %s',
+    (protocolVersion) => {
+      const request = {
+        protocolVersion,
+        clientInfo: { name: 'TestClient', version: '1.0.0' }
+      };
+
+      const check = createClientInitializationCheck(request);
+      expect(check.status).toBe('FAILURE');
+      expect(check.errorMessage).toContain('Version mismatch');
+    }
+  );
+
   it('should support custom expected spec version', () => {
     const request = {
       protocolVersion: '2024-11-05',
diff --git a/src/checks/client.ts b/src/checks/client.ts
@@ -1,4 +1,9 @@
-import { ConformanceCheck, CheckStatus } from '../types';
+import {
+  ConformanceCheck,
+  CheckStatus,
+  LATEST_SPEC_VERSION,
+  NEGOTIABLE_PROTOCOL_VERSIONS
+} from '../types';
 
 export function createServerInfoCheck(serverInfo: {
   name: string;
@@ -23,19 +28,18 @@ export function createServerInfoCheck(serverInfo: {
   };
 }
 
-// Valid MCP protocol versions
-const VALID_PROTOCOL_VERSIONS = ['2025-06-18', '2025-11-25'];
-
 export function createClientInitializationCheck(
   initializeRequest: any,
-  expectedSpecVersion: string = '2025-11-25'
+  expectedSpecVersion: string = LATEST_SPEC_VERSION
 ): ConformanceCheck {
   const protocolVersionSent = initializeRequest?.protocolVersion;
 
   // Accept known valid versions OR custom expected version (for backward compatibility)
-  const validVersions = VALID_PROTOCOL_VERSIONS.includes(expectedSpecVersion)
-    ? VALID_PROTOCOL_VERSIONS
-    : [...VALID_PROTOCOL_VERSIONS, expectedSpecVersion];
+  const validVersions = NEGOTIABLE_PROTOCOL_VERSIONS.includes(
+    expectedSpecVersion
+  )
+    ? NEGOTIABLE_PROTOCOL_VERSIONS
+    : [...NEGOTIABLE_PROTOCOL_VERSIONS, expectedSpecVersion];
   const versionMatch = validVersions.includes(protocolVersionSent);
 
   const errors: string[] = [];
diff --git a/src/index.ts b/src/index.ts
@@ -152,7 +152,8 @@ program
                 options.command,
                 scenarioName,
                 timeout,
-                outputDir
+                outputDir,
+                specVersionFilter
               );
               return {
                 scenario: scenarioName,
@@ -259,7 +260,8 @@ program
         validated.command,
         validated.scenario,
         timeout,
-        outputDir
+        outputDir,
+        specVersionFilter
       );
 
       const { overallFailure } = printClientResults(
diff --git a/src/runner/client.ts b/src/runner/client.ts
@@ -1,7 +1,7 @@
 import { spawn } from 'child_process';
 import { promises as fs } from 'fs';
 import path from 'path';
-import { ConformanceCheck } from '../types';
+import { ConformanceCheck, SpecVersion } from '../types';
 import { getScenario } from '../scenarios';
 import { createResultDir, formatPrettyChecks } from './utils';
 
@@ -17,7 +17,8 @@ async function executeClient(
   scenarioName: string,
   serverUrl: string,
   timeout: number = 30000,
-  context?: Record<string, unknown>
+  context?: Record<string, unknown>,
+  specVersion?: SpecVersion
 ): Promise<ClientExecutionResult> {
   const commandParts = command.split(' ');
   const executable = commandParts[0];
@@ -34,6 +35,9 @@ async function executeClient(
   // 3. Semantic separation: scenario identifies "which test", context provides "test data"
   const env = { ...process.env };
   env.MCP_CONFORMANCE_SCENARIO = scenarioName;
+  if (specVersion) {
+    env.MCP_CONFORMANCE_PROTOCOL_VERSION = specVersion;
+  }
   if (context) {
     // Include scenario name in context for discriminated union parsing
     env.MCP_CONFORMANCE_CONTEXT = JSON.stringify({
@@ -92,7 +96,8 @@ export async function runConformanceTest(
   clientCommand: string,
   scenarioName: string,
   timeout: number = 30000,
-  outputDir?: string
+  outputDir?: string,
+  specVersion?: SpecVersion
 ): Promise<{
   checks: ConformanceCheck[];
   clientOutput: ClientExecutionResult;
@@ -123,7 +128,8 @@ export async function runConformanceTest(
       scenarioName,
       urls.serverUrl,
       timeout,
-      urls.context
+      urls.context,
+      specVersion
     );
 
     // Print stdout/stderr if client exited with nonzero code
diff --git a/src/scenarios/client/auth/client-credentials.ts b/src/scenarios/client/auth/client-credentials.ts
@@ -4,7 +4,7 @@ import type {
   Scenario,
   ConformanceCheck,
   ScenarioUrls,
-  SpecVersion
+  ScenarioSpecTag
 } from '../../../types';
 import { createAuthServer } from './helpers/createAuthServer';
 import { createServer } from './helpers/createServer';
@@ -37,7 +37,7 @@ async function generateTestKeypair(): Promise<{
  */
 export class ClientCredentialsJwtScenario implements Scenario {
   name = 'auth/client-credentials-jwt';
-  specVersions: SpecVersion[] = ['extension'];
+  specVersions: ScenarioSpecTag[] = ['extension'];
   description =
     'Tests OAuth client_credentials flow with private_key_jwt authentication (SEP-1046)';
 
@@ -256,7 +256,7 @@ export class ClientCredentialsJwtScenario implements Scenario {
  */
 export class ClientCredentialsBasicScenario implements Scenario {
   name = 'auth/client-credentials-basic';
-  specVersions: SpecVersion[] = ['extension'];
+  specVersions: ScenarioSpecTag[] = ['extension'];
   description =
     'Tests OAuth client_credentials flow with client_secret_basic authentication';
 
diff --git a/src/scenarios/client/auth/cross-app-access.ts b/src/scenarios/client/auth/cross-app-access.ts
@@ -5,7 +5,7 @@ import type {
   Scenario,
   ConformanceCheck,
   ScenarioUrls,
-  SpecVersion
+  ScenarioSpecTag
 } from '../../../types';
 import { createAuthServer } from './helpers/createAuthServer';
 import { createServer } from './helpers/createServer';
@@ -60,7 +60,7 @@ async function createIdpIdToken(
  */
 export class CrossAppAccessCompleteFlowScenario implements Scenario {
   name = 'auth/cross-app-access-complete-flow';
-  specVersions: SpecVersion[] = ['extension'];
+  specVersions: ScenarioSpecTag[] = ['extension'];
   description =
     'Tests complete SEP-990 flow: token exchange + JWT bearer grant (Enterprise Managed OAuth)';
 
diff --git a/src/scenarios/client/auth/offline-access.ts b/src/scenarios/client/auth/offline-access.ts
@@ -1,5 +1,9 @@
 import type { Scenario, ConformanceCheck } from '../../../types';
-import { ScenarioUrls, SpecVersion } from '../../../types';
+import {
+  ScenarioUrls,
+  SpecVersion,
+  DRAFT_PROTOCOL_VERSION
+} from '../../../types';
 import { createAuthServer } from './helpers/createAuthServer';
 import { createServer } from './helpers/createServer';
 import { ServerLifecycle } from './helpers/serverLifecycle';
@@ -23,7 +27,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier';
  */
 export class OfflineAccessScopeScenario implements Scenario {
   name = 'auth/offline-access-scope';
-  specVersions: SpecVersion[] = ['draft'];
+  specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
   description =
     'Tests that a client that wants a refresh token handles offline_access scope and refresh_token grant type when AS supports them (SEP-2207)';
 
@@ -227,7 +231,7 @@ export class OfflineAccessScopeScenario implements Scenario {
  */
 export class OfflineAccessNotSupportedScenario implements Scenario {
   name = 'auth/offline-access-not-supported';
-  specVersions: SpecVersion[] = ['draft'];
+  specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
   description =
     'Tests that client does not request offline_access when AS does not list it in scopes_supported (SEP-2207)';
 
diff --git a/src/scenarios/client/auth/resource-mismatch.ts b/src/scenarios/client/auth/resource-mismatch.ts
@@ -1,5 +1,9 @@
 import type { Scenario, ConformanceCheck } from '../../../types.js';
-import { ScenarioUrls, SpecVersion } from '../../../types.js';
+import {
+  ScenarioUrls,
+  SpecVersion,
+  DRAFT_PROTOCOL_VERSION
+} from '../../../types.js';
 import { createAuthServer } from './helpers/createAuthServer.js';
 import { createServer } from './helpers/createServer.js';
 import { ServerLifecycle } from './helpers/serverLifecycle.js';
@@ -27,7 +31,7 @@ import { MockTokenVerifier } from './helpers/mockTokenVerifier.js';
  */
 export class ResourceMismatchScenario implements Scenario {
   name = 'auth/resource-mismatch';
-  specVersions: SpecVersion[] = ['draft'];
+  specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
   description =
     'Tests that client rejects when PRM resource does not match server URL';
   allowClientError = true;
diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts
@@ -3,7 +3,9 @@ import {
   Scenario,
   ScenarioUrls,
   ConformanceCheck,
-  SpecVersion
+  SpecVersion,
+  LATEST_SPEC_VERSION,
+  NEGOTIABLE_PROTOCOL_VERSIONS
 } from '../../types';
 import { clientChecks } from '../../checks/index';
 
@@ -117,11 +119,10 @@ export class InitializeScenario implements Scenario {
     this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
 
     // Echo back client's version if valid, otherwise use latest
-    const VALID_VERSIONS = ['2025-06-18', '2025-11-25'];
     const clientVersion = initializeRequest?.protocolVersion;
-    const responseVersion = VALID_VERSIONS.includes(clientVersion)
+    const responseVersion = NEGOTIABLE_PROTOCOL_VERSIONS.includes(clientVersion)
       ? clientVersion
-      : '2025-11-25';
+      : LATEST_SPEC_VERSION;
 
     const response = {
       jsonrpc: '2.0',
diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts
@@ -2,7 +2,11 @@ import {
   Scenario,
   ClientScenario,
   ClientScenarioForAuthorizationServer,
-  SpecVersion
+  SpecVersion,
+  ScenarioSpecTag,
+  DATED_SPEC_VERSIONS,
+  DRAFT_PROTOCOL_VERSION,
+  LATEST_SPEC_VERSION
 } from '../types';
 import { InitializeScenario } from './client/initialize';
 import { ToolsCallScenario } from './client/tools_call';
@@ -256,51 +260,69 @@ export function listDraftScenarios(): string[] {
 export { listMetadataScenarios };
 
 // All valid spec versions, used by the CLI to validate --spec-version input.
+// 'extension' is intentionally excluded — extension scenarios are off-timeline
+// and selected via `--suite extensions`, not `--spec-version`.
 export const ALL_SPEC_VERSIONS: SpecVersion[] = [
-  '2025-03-26',
-  '2025-06-18',
-  '2025-11-25',
-  'draft',
-  'extension'
+  ...DATED_SPEC_VERSIONS,
+  DRAFT_PROTOCOL_VERSION
 ];
 
 export function resolveSpecVersion(value: string): SpecVersion {
+  if (value === 'draft') return DRAFT_PROTOCOL_VERSION;
   if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
     return value as SpecVersion;
   }
   console.error(`Unknown spec version: ${value}`);
-  console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
+  console.error(
+    `Valid versions: ${ALL_SPEC_VERSIONS.join(', ')} (or 'draft' as an alias for ${DRAFT_PROTOCOL_VERSION})`
+  );
   process.exit(1);
 }
 
+// The draft version selects everything in the latest dated release plus
+// scenarios tagged draft-only, so SEP authors can run the full suite against an
+// SDK tracking the in-progress spec without retagging core scenarios.
+function matchesSpecVersion(
+  scenario: { specVersions: ScenarioSpecTag[] },
+  version: SpecVersion
+): boolean {
+  if (version === DRAFT_PROTOCOL_VERSION) {
+    return (
+      scenario.specVersions.includes(DRAFT_PROTOCOL_VERSION) ||
+      scenario.specVersions.includes(LATEST_SPEC_VERSION)
+    );
+  }
+  return scenario.specVersions.includes(version);
+}
+
 export function listScenariosForSpec(version: SpecVersion): string[] {
   return scenariosList
-    .filter((s) => s.specVersions.includes(version))
+    .filter((s) => matchesSpecVersion(s, version))
     .map((s) => s.name);
 }
 
 export function listClientScenariosForSpec(version: SpecVersion): string[] {
   return allClientScenariosList
-    .filter((s) => s.specVersions.includes(version))
+    .filter((s) => matchesSpecVersion(s, version))
     .map((s) => s.name);
 }
 
 export function listClientScenariosForAuthorizationServerForSpec(
   version: SpecVersion
 ): string[] {
   return allClientScenariosListForAuthorizationServer
-    .filter((s) => s.specVersions.includes(version))
+    .filter((s) => matchesSpecVersion(s, version))
     .map((s) => s.name);
 }
 
 export function getScenarioSpecVersions(
   name: string
-): SpecVersion[] | undefined {
+): ScenarioSpecTag[] | undefined {
   return (
     scenarios.get(name)?.specVersions ??
     clientScenarios.get(name)?.specVersions ??
     clientScenariosForAuthorizationServer.get(name)?.specVersions
   );
 }
 
-export type { SpecVersion };
+export type { SpecVersion, ScenarioSpecTag };
diff --git a/src/scenarios/server/resources.ts b/src/scenarios/server/resources.ts
@@ -2,7 +2,12 @@
  * Resources test scenarios for MCP servers
  */
 
-import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
+import {
+  ClientScenario,
+  ConformanceCheck,
+  SpecVersion,
+  DRAFT_PROTOCOL_VERSION
+} from '../../types';
 import { connectToServer } from './client-helper';
 import {
   TextResourceContents,
@@ -438,7 +443,7 @@ Example request:
 
 export class ResourcesNotFoundErrorScenario implements ClientScenario {
   name = 'sep-2164-resource-not-found';
-  specVersions: SpecVersion[] = ['draft'];
+  specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
   description = `Test error handling for non-existent resources (SEP-2164).
 
 **Server Implementation Requirements:**
diff --git a/src/scenarios/spec-version.test.ts b/src/scenarios/spec-version.test.ts
diff --git a/src/tier-check/checks/test-conformance-results.ts b/src/tier-check/checks/test-conformance-results.ts
diff --git a/src/tier-check/output.ts b/src/tier-check/output.ts
diff --git a/src/tier-check/types.ts b/src/tier-check/types.ts
diff --git a/src/types.ts b/src/types.ts
