feat: add MCP server mocks (closes #363) (#387)

* feat: add MCP server mocks #363

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: harden MCP mock config handling #363

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: spboyer

README.md

@@ -943,7 +943,7 @@ skills/                Example skills
 ```yaml
 name: my-eval
 skill: my-skill
-schemaVersion: "1.0"
+schemaVersion: "1.1"
 version: "1.0"
 
 config:
@@ -988,6 +988,26 @@ hooks:
       exit_codes: [0]
       error_on_fail: false
 
+mcp_mocks:
+  - name: github
+    tools:
+      list_issues:
+        input_schema:
+          type: object
+          required: [owner, repo]
+        responses:
+          - match:
+              owner: microsoft
+              repo: waza
+            return:
+              issues:
+                - number: 363
+                  title: MCP server mocks for hermetic eval
+          - match_regex:
+              repo: "^waza-.*"
+            return:
+              issues: []
+
 graders:
   - type: text
     name: pattern_check
@@ -1017,6 +1037,19 @@ tasks:
 
 `schemaVersion` uses `MAJOR.MINOR` format and defaults to `1.0` when omitted for backward compatibility. Readers allow same-major minor additions with warnings for unknown fields, but reject different majors with a hint to run `waza migrate <file>`.
 
+### MCP Mock Servers
+
+Use top-level `mcp_mocks` with `schemaVersion: "1.1"` for deterministic Copilot SDK evals that need MCP tools without live services. Waza launches each mock as a local stdio MCP server, so CI runs do not need network ports, external credentials, or real service state.
+
+```yaml
+schemaVersion: "1.1"
+mcp_mocks:
+  - name: github
+    fixtures: fixtures/mcp/github
+```
+
+Inline responses support exact full-argument matching (`match`), JSON Schema matching (`match_schema`), and per-field regex matching (`match_regex`). Unknown tools and unmatched calls fail loudly with an MCP tool error that names the missing fixture.
+
 ### Custom Input Variables
 
 Use the `inputs` section to define key-value variables available throughout your evaluation as `{{.Vars.key}}`:

cmd/waza/cmd_mcp_mock.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/microsoft/waza/internal/mcpmock"
+	"github.com/spf13/cobra"
+)
+
+func newMCPMockCommand() *cobra.Command {
+	var configBase64 string
+	var configFile string
+	cmd := &cobra.Command{
+		Use:    "__mcp-mock",
+		Short:  "Run an internal MCP mock server",
+		Hidden: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			data, err := readMCPMockConfig(configBase64, configFile)
+			if err != nil {
+				return err
+			}
+			var cfg mcpmock.Config
+			if err := json.Unmarshal(data, &cfg); err != nil {
+				return fmt.Errorf("parse mock config: %w", err)
+			}
+			logger := slog.New(slog.NewTextHandler(os.Stderr, nil))
+			mcpmock.ServeStdio(context.Background(), &cfg, os.Stdin, os.Stdout, logger)
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&configBase64, "config-base64", "", "base64-encoded MCP mock config")
+	cmd.Flags().StringVar(&configFile, "config-file", "", "path to MCP mock config JSON")
+	return cmd
+}
+
+func readMCPMockConfig(configBase64, configFile string) ([]byte, error) {
+	switch {
+	case configFile != "":
+		data, err := os.ReadFile(configFile)
+		if err != nil {
+			return nil, fmt.Errorf("read mock config file: %w", err)
+		}
+		if err := os.Remove(configFile); err != nil {
+			return nil, fmt.Errorf("remove mock config file: %w", err)
+		}
+		return data, nil
+	case configBase64 != "":
+		data, err := base64.StdEncoding.DecodeString(configBase64)
+		if err != nil {
+			return nil, fmt.Errorf("decode mock config: %w", err)
+		}
+		return data, nil
+	default:
+		return nil, fmt.Errorf("--config-file or --config-base64 is required")
+	}
+}

cmd/waza/root.go

@@ -71,6 +71,7 @@ performance against predefined test cases.`,
 	cmd.AddCommand(newUpdateCommand())
 	cmd.AddCommand(newSpecCommand())
 	cmd.AddCommand(newMigrateCommand())
+	cmd.AddCommand(newMCPMockCommand())
 
 	return cmd
 }

docs/INTEGRATION-TESTING.md

@@ -47,6 +47,51 @@ config:
       args: ["-y", "@azure/mcp", "server", "start"]
 ```
 
+### Hermetic MCP Mock Servers
+
+For CI-safe Copilot SDK evals, prefer top-level `mcp_mocks` over live `config.mcp_servers`. Mock servers run as local stdio MCP servers managed by waza, so they do not bind ports, make network calls, or require service credentials. Because `mcp_mocks` is an additive eval schema field, set `schemaVersion: "1.1"`.
+
+```yaml
+name: github-triage
+skill: issue-triage
+schemaVersion: "1.1"
+version: "1.0"
+
+config:
+  trials_per_task: 1
+  executor: copilot-sdk
+  model: claude-sonnet-4.6
+  timeout_seconds: 300
+
+mcp_mocks:
+  - name: github
+    tools:
+      list_issues:
+        input_schema:
+          type: object
+          properties:
+            owner: { type: string }
+            repo: { type: string }
+          required: [owner, repo]
+        responses:
+          - match:
+              owner: microsoft
+              repo: waza
+            return:
+              issues:
+                - number: 363
+                  title: MCP server mocks for hermetic eval
+          - match_schema:
+              type: object
+              required: [owner, repo]
+            error: "No matching issue fixture"
+
+tasks:
+  - tasks/*.yaml
+```
+
+Responses are matched in order. Use `match` for exact full-argument fixtures, `match_schema` for JSON Schema matching, and `match_regex` for per-field regular expressions. Unknown tools or unmatched calls fail with a clear MCP tool error so missing fixtures do not pass silently.
+
 ### CLI Override
 
 You can override the model and runtime options at launch:

internal/copilotconfig/mcp.go

@@ -1,21 +1,38 @@
 package copilotconfig
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 
 	copilot "github.com/github/copilot-sdk/go"
 	"github.com/go-viper/mapstructure/v2"
+	"github.com/microsoft/waza/internal/mcpmock"
+	"github.com/microsoft/waza/internal/models"
 )
 
 // ConvertMCPServers converts eval YAML mcp_servers entries into Copilot SDK MCP
 // configs. Invalid entries are skipped after emitting a warning through warnf.
 func ConvertMCPServers(serverConfigs map[string]any, warnf func(string, ...any)) map[string]copilot.MCPServerConfig {
+	return ConvertMCPServersWithMocks(serverConfigs, nil, "", warnf)
+}
+
+// ConvertMCPServersWithMocks converts eval YAML mcp_servers entries and
+// mcp_mocks entries into Copilot SDK MCP configs. Mock servers are exposed as
+// hermetic stdio MCP servers backed by this waza binary.
+func ConvertMCPServersWithMocks(serverConfigs map[string]any, mocks []models.MCPMockConfig, baseDir string, warnf func(string, ...any)) map[string]copilot.MCPServerConfig {
+	if warnf == nil {
+		warnf = func(string, ...any) {}
+	}
 	if len(serverConfigs) == 0 {
-		return nil
+		if len(mocks) == 0 {
+			return nil
+		}
 	}
 
-	result := make(map[string]copilot.MCPServerConfig, len(serverConfigs))
+	result := make(map[string]copilot.MCPServerConfig, len(serverConfigs)+len(mocks))
 	for name, cfg := range serverConfigs {
 		cfgMap, ok := cfg.(map[string]any)
 		if !ok {
@@ -44,9 +61,92 @@ func ConvertMCPServers(serverConfigs map[string]any, warnf func(string, ...any))
 		}
 	}
 
+	for _, mock := range mocks {
+		mockName := strings.TrimSpace(mock.Name)
+		if mockName != "" {
+			delete(result, mockName)
+		}
+		cfg, err := mcpmock.FromEvalConfig(mock, baseDir)
+		if err != nil {
+			warnf("Warning: mcp_mock %q config is invalid: %v, skipping\n", mock.Name, err)
+			continue
+		}
+		stdio, err := mockServerConfig(*cfg)
+		if err != nil {
+			warnf("Warning: mcp_mock %q could not be configured: %v, skipping\n", cfg.Name, err)
+			continue
+		}
+		result[cfg.Name] = stdio
+	}
+
+	if len(result) == 0 {
+		return nil
+	}
 	return result
 }
 
+func mockServerConfig(cfg mcpmock.Config) (copilot.MCPStdioServerConfig, error) {
+	data, err := json.Marshal(cfg)
+	if err != nil {
+		return copilot.MCPStdioServerConfig{}, fmt.Errorf("marshal mock config: %w", err)
+	}
+	configFile, err := writeMockConfigFile(cfg.Name, data)
+	if err != nil {
+		return copilot.MCPStdioServerConfig{}, err
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return copilot.MCPStdioServerConfig{}, fmt.Errorf("resolve waza executable: %w", err)
+	}
+	return copilot.MCPStdioServerConfig{
+		Command: exe,
+		Args:    []string{"__mcp-mock", "--config-file", configFile},
+		Env: map[string]string{
+			"WAZA_NO_UPDATE_CHECK": "1",
+		},
+	}, nil
+}
+
+func writeMockConfigFile(name string, data []byte) (string, error) {
+	safeName := strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z':
+			return r
+		case r >= 'A' && r <= 'Z':
+			return r
+		case r >= '0' && r <= '9':
+			return r
+		case r == '-' || r == '_':
+			return r
+		default:
+			return '-'
+		}
+	}, name)
+	if safeName == "" {
+		safeName = "mock"
+	}
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("waza-mcp-mock-%s-*.json", safeName))
+	file, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path))
+	if err != nil {
+		return "", fmt.Errorf("create mock config file: %w", err)
+	}
+	if err := file.Chmod(0600); err != nil {
+		_ = file.Close()
+		_ = os.Remove(file.Name())
+		return "", fmt.Errorf("secure mock config file: %w", err)
+	}
+	if _, err := file.Write(data); err != nil {
+		_ = file.Close()
+		_ = os.Remove(file.Name())
+		return "", fmt.Errorf("write mock config file: %w", err)
+	}
+	if err := file.Close(); err != nil {
+		_ = os.Remove(file.Name())
+		return "", fmt.Errorf("close mock config file: %w", err)
+	}
+	return file.Name(), nil
+}
+
 func decode(input map[string]any, output any) error {
 	decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
 		Result:           output,

internal/copilotconfig/mcp_test.go

@@ -0,0 +1,75 @@
+package copilotconfig
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	copilot "github.com/github/copilot-sdk/go"
+	"github.com/microsoft/waza/internal/mcpmock"
+	"github.com/microsoft/waza/internal/models"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertMCPServersWithMocks_AddsHermeticStdioServer(t *testing.T) {
+	var warnings []string
+	servers := ConvertMCPServersWithMocks(nil, []models.MCPMockConfig{{
+		Name: "github",
+		Tools: map[string]models.MCPMockTool{
+			"list_issues": {
+				Responses: []models.MCPMockResponse{{Match: map[string]any{"owner": "octocat"}, Return: map[string]any{"issues": []any{}}}},
+			},
+		},
+	}}, t.TempDir(), func(format string, args ...any) {
+		warnings = append(warnings, fmtString(format, args...))
+	})
+
+	require.Empty(t, warnings)
+	require.Contains(t, servers, "github")
+	stdio, ok := servers["github"].(copilot.MCPStdioServerConfig)
+	require.True(t, ok)
+	require.NotEmpty(t, stdio.Command)
+	require.Equal(t, "1", stdio.Env["WAZA_NO_UPDATE_CHECK"])
+	require.Len(t, stdio.Args, 3)
+	require.Equal(t, "__mcp-mock", stdio.Args[0])
+	require.Equal(t, "--config-file", stdio.Args[1])
+
+	data, err := os.ReadFile(stdio.Args[2])
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = os.Remove(stdio.Args[2]) })
+	var cfg mcpmock.Config
+	require.NoError(t, json.Unmarshal(data, &cfg))
+	require.Equal(t, "github", cfg.Name)
+	require.Contains(t, cfg.Tools, "list_issues")
+}
+
+func TestConvertMCPServersWithMocks_PreservesRegularServers(t *testing.T) {
+	servers := ConvertMCPServersWithMocks(map[string]any{
+		"regular": map[string]any{"type": "stdio", "command": "echo"},
+	}, nil, "", nil)
+
+	require.Contains(t, servers, "regular")
+	_, ok := servers["regular"].(copilot.MCPStdioServerConfig)
+	require.True(t, ok)
+}
+
+func TestConvertMCPServersWithMocks_InvalidMockDisablesLiveServerFallback(t *testing.T) {
+	var warnings []string
+	servers := ConvertMCPServersWithMocks(map[string]any{
+		"github": map[string]any{"type": "stdio", "command": "echo"},
+	}, []models.MCPMockConfig{{
+		Name:     " github ",
+		Fixtures: "missing",
+	}}, t.TempDir(), func(format string, args ...any) {
+		warnings = append(warnings, fmtString(format, args...))
+	})
+
+	require.NotEmpty(t, warnings)
+	require.NotContains(t, servers, "github")
+}
+
+func fmtString(format string, args ...any) string {
+	return strings.TrimSpace(fmt.Sprintf(format, args...))
+}