feat: Tool name format checks (SEP-986) (#240)

Author: nbarbettini

src/scenarios/server/tools.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from 'vitest';
+import { buildToolsNameFormatCheck, validateToolNameFormat } from './tools.js';
+
+describe('validateToolNameFormat', () => {
+  it('accepts a typical snake_case name', () => {
+    expect(validateToolNameFormat('test_simple_text')).toBeNull();
+  });
+
+  it('accepts all allowed character classes', () => {
+    expect(validateToolNameFormat('Aa0_.-/')).toBeNull();
+  });
+
+  it('accepts a single-character name (lower length boundary)', () => {
+    expect(validateToolNameFormat('a')).toBeNull();
+  });
+
+  it('accepts a 64-character name (upper length boundary)', () => {
+    expect(validateToolNameFormat('a'.repeat(64))).toBeNull();
+  });
+
+  it('rejects an empty name', () => {
+    expect(validateToolNameFormat('')).toMatch(/length 0 is outside/);
+  });
+
+  it('rejects a 65-character name', () => {
+    expect(validateToolNameFormat('a'.repeat(65))).toMatch(
+      /length 65 is outside/
+    );
+  });
+
+  it.each([
+    ['space', 'bad name'],
+    ['colon', 'bad:name'],
+    ['at sign', 'bad@name'],
+    ['unicode', 'bad\u00e9name'],
+    ['backslash', 'bad\\name'],
+    ['plus', 'bad+name']
+  ])('rejects a name with a disallowed character (%s)', (_label, name) => {
+    expect(validateToolNameFormat(name)).toMatch(/contains characters outside/);
+  });
+});
+
+describe('buildToolsNameFormatCheck', () => {
+  it('returns INFO when tools is undefined', () => {
+    const check = buildToolsNameFormatCheck(undefined);
+    expect(check.status).toBe('INFO');
+    expect(check.id).toBe('tools-name-format');
+    expect(check.details).toEqual({ toolCount: 0 });
+  });
+
+  it('returns INFO when tools is an empty array', () => {
+    const check = buildToolsNameFormatCheck([]);
+    expect(check.status).toBe('INFO');
+    expect(check.errorMessage).toBe('No tools advertised; nothing to validate');
+  });
+
+  it('returns SUCCESS when all tool names are valid', () => {
+    const check = buildToolsNameFormatCheck([
+      { name: 'test_simple_text' },
+      { name: 'namespace/tool-v1.2' }
+    ]);
+    expect(check.status).toBe('SUCCESS');
+    expect(check.errorMessage).toBeUndefined();
+    expect(check.details).toMatchObject({
+      toolCount: 2,
+      results: {
+        test_simple_text: 'valid',
+        'namespace/tool-v1.2': 'valid'
+      }
+    });
+  });
+
+  it('returns FAILURE with per-tool details when some names are invalid', () => {
+    const check = buildToolsNameFormatCheck([
+      { name: 'good_tool' },
+      { name: 'bad name with spaces' },
+      { name: 'a'.repeat(65) }
+    ]);
+
+    expect(check.status).toBe('FAILURE');
+    expect(check.errorMessage).toContain(
+      '2 tool name(s) violate SEP-986 format'
+    );
+
+    const results = (check.details as { results: Record<string, string> })
+      .results;
+    expect(results['good_tool']).toBe('valid');
+    expect(results['bad name with spaces']).toMatch(/^invalid: /);
+    expect(results['a'.repeat(65)]).toMatch(/^invalid: length 65/);
+  });
+
+  it('flags a tool whose name is not a string', () => {
+    const check = buildToolsNameFormatCheck([{ name: 123 as unknown }]);
+    expect(check.status).toBe('FAILURE');
+    const results = (check.details as { results: Record<string, string> })
+      .results;
+    expect(results['<tool[0] missing name>']).toBe(
+      'invalid: name is not a string'
+    );
+  });
+
+  it('includes both MCP-Tools-List and SEP-986 spec references', () => {
+    const check = buildToolsNameFormatCheck([{ name: 'ok' }]);
+    const ids = check.specReferences?.map((r) => r.id);
+    expect(ids).toEqual(['MCP-Tools-List', 'SEP-986']);
+  });
+});

src/scenarios/server/tools.ts

@@ -11,6 +11,83 @@ import {
   Progress
 } from '@modelcontextprotocol/sdk/types.js';
 
+const TOOL_NAME_PATTERN = /^[A-Za-z0-9_./-]+$/;
+const TOOL_NAME_MAX_LENGTH = 64;
+
+const TOOLS_NAME_FORMAT_SPEC_REFS = [
+  {
+    id: 'MCP-Tools-List',
+    url: 'https://modelcontextprotocol.io/specification/2025-11-25/server/tools#listing-tools'
+  },
+  {
+    id: 'SEP-986',
+    url: 'https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/SEP/SEP-986.md'
+  }
+];
+
+export function validateToolNameFormat(name: string): string | null {
+  if (name.length < 1 || name.length > TOOL_NAME_MAX_LENGTH) {
+    return `length ${name.length} is outside 1-${TOOL_NAME_MAX_LENGTH}`;
+  }
+  if (!TOOL_NAME_PATTERN.test(name)) {
+    return 'contains characters outside [A-Za-z0-9_./-]';
+  }
+  return null;
+}
+
+export function buildToolsNameFormatCheck(
+  tools: ReadonlyArray<{ name?: unknown }> | undefined
+): ConformanceCheck {
+  const timestamp = new Date().toISOString();
+  const baseCheck = {
+    id: 'tools-name-format',
+    name: 'ToolsNameFormat',
+    description: 'Tool names are 1-64 characters and match ^[A-Za-z0-9_./-]+$',
+    specReferences: TOOLS_NAME_FORMAT_SPEC_REFS,
+    timestamp
+  };
+
+  if (!Array.isArray(tools) || tools.length === 0) {
+    return {
+      ...baseCheck,
+      status: 'INFO',
+      errorMessage: 'No tools advertised; nothing to validate',
+      details: { toolCount: 0 }
+    };
+  }
+
+  const toolResults: Record<string, string> = {};
+  const violations: string[] = [];
+
+  tools.forEach((tool, index) => {
+    const name = typeof tool.name === 'string' ? tool.name : '';
+    const key = name || `<tool[${index}] missing name>`;
+    const reason =
+      typeof tool.name === 'string'
+        ? validateToolNameFormat(tool.name)
+        : 'name is not a string';
+    if (reason) {
+      toolResults[key] = `invalid: ${reason}`;
+      violations.push(`${key}: ${reason}`);
+    } else {
+      toolResults[key] = 'valid';
+    }
+  });
+
+  return {
+    ...baseCheck,
+    status: violations.length === 0 ? 'SUCCESS' : 'FAILURE',
+    errorMessage:
+      violations.length > 0
+        ? `${violations.length} tool name(s) violate SEP-986 format: ${violations.join('; ')}`
+        : undefined,
+    details: {
+      toolCount: tools.length,
+      results: toolResults
+    }
+  };
+}
+
 export class ToolsListScenario implements ClientScenario {
   name = 'tools-list';
   specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];
@@ -23,7 +100,7 @@ export class ToolsListScenario implements ClientScenario {
 **Requirements**:
 - Return array of all available tools
 - Each tool MUST have:
-  - \`name\` (string)
+  - \`name\` (string, 1-64 chars, matching \`^[A-Za-z0-9_./-]+$\`)
   - \`description\` (string)
   - \`inputSchema\` (valid JSON Schema object)`;
 
@@ -72,6 +149,10 @@ export class ToolsListScenario implements ClientScenario {
         }
       });
 
+      // Validate tool name format per SEP-986:
+      // names MUST be 1-64 chars matching ^[A-Za-z0-9_./-]+$
+      checks.push(buildToolsNameFormatCheck(result.tools));
+
       await connection.close();
     } catch (error) {
       checks.push({