Fail reliably when the conformance server dies

The cleanup path awaited a child exit event that could already have
fired, hanging the promise chain and letting Node exit 0 after a
server crash — a crashed HTTP transport passed the check. The exit
promise is now armed at spawn so cleanup always resolves, a pre-spawn
probe rejects a port already occupied by another process instead of
running conformance against it, and the doctor's spawned server is
pinned to the stdio transport so an exported MCP_TRANSPORT cannot
skew it. Also caps the mcp-checks CI job at 15 minutes and documents
the conformance port and its PORT override.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: alistair3149

.github/workflows/ci.yml

@@ -25,6 +25,7 @@ jobs:
 
   mcp-checks:
     runs-on: ubuntu-24.04-arm
+    timeout-minutes: 15
     steps:
     - uses: actions/checkout@v7
 

docs/testing.md

@@ -78,7 +78,7 @@ Token-free structural checks against the built server: a stdio doctor sweep plus
 npm run check:mcp
 ```
 
-Both use the minimal wiki configuration in `scripts/mcp-checks.config.json`, so results do not depend on your local `config.json`. CI runs the same script on every pull request, and additionally diffs the tool surface against the PR base — a removed or renamed tool fails the job, while description changes only show up in the report.
+Both use the minimal wiki configuration in `scripts/mcp-checks.config.json`, so results do not depend on your local `config.json`. The conformance run binds `127.0.0.1:3117`; set `PORT` to use a different port. CI runs the same script on every pull request, and additionally diffs the tool surface against the PR base — a removed or renamed tool fails the job, while description changes only show up in the report.
 
 ## MCP Inspector CLI (integration tests)
 

scripts/mcp-checks.cjs

@@ -37,6 +37,8 @@ function runDoctor() {
 			REPO_ROOT,
 			'--env',
 			`CONFIG=${CONFIG_PATH}`,
+			'--env',
+			'MCP_TRANSPORT=stdio',
 			'--no-telemetry',
 			'--quiet',
 		],
@@ -78,25 +80,37 @@ function runDoctor() {
 	return true;
 }
 
+async function respondsOverHttp() {
+	try {
+		// Any HTTP response, including an error status, means something
+		// is accepting connections on the port.
+		await fetch(SERVER_URL, { method: 'GET', signal: AbortSignal.timeout(2000) });
+		return true;
+	} catch {
+		return false;
+	}
+}
+
 async function waitForServer(child) {
 	const deadline = Date.now() + STARTUP_TIMEOUT_MS;
 	while (Date.now() < deadline) {
 		if (child.exitCode !== null) {
 			return false;
 		}
-		try {
-			// Any HTTP response, including an error status, means the
-			// transport is accepting connections.
-			await fetch(SERVER_URL, { method: 'GET' });
+		if (await respondsOverHttp()) {
 			return true;
-		} catch {
-			await new Promise((resolve) => setTimeout(resolve, 250));
 		}
+		await new Promise((resolve) => setTimeout(resolve, 250));
 	}
 	return false;
 }
 
 async function runConformance() {
+	if (await respondsOverHttp()) {
+		console.error(`Port ${PORT} is already in use by another process — set PORT to a free port.`);
+		return false;
+	}
+
 	console.log(`\nStarting HTTP transport on port ${PORT}...`);
 	const serverLog = [];
 	const child = spawn(process.execPath, [DIST_ENTRY], {
@@ -111,9 +125,12 @@ async function runConformance() {
 	});
 	child.stdout.on('data', (chunk) => serverLog.push(chunk));
 	child.stderr.on('data', (chunk) => serverLog.push(chunk));
+	// Armed before the child can possibly exit, so the finally block's
+	// await resolves even when the child is already dead by then.
+	const exited = new Promise((resolve) => child.once('exit', resolve));
 
 	try {
-		if (!(await waitForServer(child))) {
+		if (!(await waitForServer(child)) || child.exitCode !== null) {
 			console.error('HTTP transport did not become reachable:');
 			console.error(serverLog.join(''));
 			return false;
@@ -142,7 +159,7 @@ async function runConformance() {
 		return true;
 	} finally {
 		child.kill('SIGTERM');
-		await new Promise((resolve) => child.once('exit', resolve));
+		await exited;
 	}
 }