fix(config): validate per-case mcp overrides

Author: roark47

internal/config/mcp_merge.go

@@ -3,6 +3,12 @@ package config
 import (
 	"errors"
 	"fmt"
+	"strings"
+)
+
+const (
+	mcpModeMocked              = "mocked"
+	builtinFilesystemMCPServer = "filesystem"
 )
 
 // MergeCaseMCP computes the effective MCP configuration for a case by merging
@@ -29,6 +35,9 @@ func MergeCaseMCP(evalMCP MCPConfig, caseMCP MCPConfig) (MCPConfig, error) {
 
 	indexByName := make(map[string]int, len(merged.Servers))
 	for i, server := range merged.Servers {
+		if strings.TrimSpace(server.Name) == "" {
+			return MCPConfig{}, errors.New("eval-level mcp server name is required")
+		}
 		if _, exists := indexByName[server.Name]; exists {
 			return MCPConfig{}, fmt.Errorf("duplicate eval-level mcp server name %q", server.Name)
 		}
@@ -45,9 +54,12 @@ func MergeCaseMCP(evalMCP MCPConfig, caseMCP MCPConfig) (MCPConfig, error) {
 		}
 		seenCaseNames[server.Name] = struct{}{}
 
-		if server.Mode != "mocked" {
+		if server.Mode != mcpModeMocked {
 			return MCPConfig{}, fmt.Errorf("case-level mcp server %q must use mode: mocked", server.Name)
 		}
+		if server.Name != builtinFilesystemMCPServer && strings.TrimSpace(server.ConfigRef) == "" {
+			return MCPConfig{}, fmt.Errorf("case-level mcp server %q mocked mode requires config_ref", server.Name)
+		}
 
 		cloned := cloneMCPServer(server)
 		if idx, ok := indexByName[server.Name]; ok {

internal/config/mcp_merge_test.go

@@ -110,8 +110,8 @@ func TestMergeCaseMCP_Errors(t *testing.T) {
 			name:    "duplicate case server names",
 			evalMCP: MCPConfig{},
 			caseMCP: MCPConfig{Servers: []MCPServer{
-				{Name: "svc", Mode: "mocked"},
-				{Name: "svc", Mode: "mocked"},
+				{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/open.yaml"},
+				{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/closed.yaml"},
 			}},
 			errMsg: "duplicate case-level mcp server name",
 		},
@@ -121,13 +121,27 @@ func TestMergeCaseMCP_Errors(t *testing.T) {
 			caseMCP: MCPConfig{Servers: []MCPServer{{Name: "svc", Mode: "real"}}},
 			errMsg:  "must use mode: mocked",
 		},
+		{
+			name:    "case server missing config ref",
+			evalMCP: MCPConfig{},
+			caseMCP: MCPConfig{Servers: []MCPServer{{Name: "svc", Mode: "mocked"}}},
+			errMsg:  "mocked mode requires config_ref",
+		},
+		{
+			name: "empty eval server name",
+			evalMCP: MCPConfig{Servers: []MCPServer{
+				{Mode: "mocked", ConfigRef: "evals/fixtures/mcp/default.yaml"},
+			}},
+			caseMCP: MCPConfig{Servers: []MCPServer{{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/open.yaml"}}},
+			errMsg:  "eval-level mcp server name is required",
+		},
 		{
 			name: "duplicate eval server names",
 			evalMCP: MCPConfig{Servers: []MCPServer{
 				{Name: "svc", Mode: "mocked"},
 				{Name: "svc", Mode: "mocked"},
 			}},
-			caseMCP: MCPConfig{Servers: []MCPServer{{Name: "svc", Mode: "mocked"}}},
+			caseMCP: MCPConfig{Servers: []MCPServer{{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/open.yaml"}}},
 			errMsg:  "duplicate eval-level mcp server name",
 		},
 	}

internal/config/validator.go

@@ -180,9 +180,12 @@ func validateCaseMCP(mcpCfg MCPConfig) []string {
 		}
 		seen[server.Name] = struct{}{}
 
-		if server.Mode != "mocked" {
+		if server.Mode != mcpModeMocked {
 			errs = append(errs, fmt.Sprintf("mcp.servers[%d] (%q) must use mode: mocked at case level", i, server.Name))
 		}
+		if server.Mode == mcpModeMocked && server.Name != builtinFilesystemMCPServer && strings.TrimSpace(server.ConfigRef) == "" {
+			errs = append(errs, fmt.Sprintf("mcp.servers[%d] (%q) mocked mode requires config_ref", i, server.Name))
+		}
 		if server.Transport != "" && server.Transport != "stdio" {
 			errs = append(errs, fmt.Sprintf("mcp.servers[%d] (%q) mocked mode only supports stdio transport", i, server.Name))
 		}

internal/config/validator_test.go

@@ -920,6 +920,17 @@ func TestValidator_ValidateCaseConfig(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "valid case-level filesystem mocked MCP override without config_ref",
+			cfg: &CaseConfig{
+				ID:    "test-case",
+				Input: Input{Prompt: "Say hello"},
+				MCP: MCPConfig{Servers: []MCPServer{
+					{Name: "filesystem", Mode: "mocked"},
+				}},
+			},
+			wantErr: false,
+		},
 		{
 			name: "case-level agent_judge with valid context",
 			cfg: &CaseConfig{
@@ -969,13 +980,25 @@ func TestValidator_ValidateCaseConfig(t *testing.T) {
 			errMsg:  "mcp.servers[0].name is required",
 		},
 		{
-			name: "case-level MCP duplicate names",
+			name: "case-level MCP missing config_ref",
 			cfg: &CaseConfig{
 				ID:    "test-case",
 				Input: Input{Prompt: "Say hello"},
 				MCP: MCPConfig{Servers: []MCPServer{
 					{Name: "svc", Mode: "mocked"},
-					{Name: "svc", Mode: "mocked"},
+				}},
+			},
+			wantErr: true,
+			errMsg:  "mocked mode requires config_ref",
+		},
+		{
+			name: "case-level MCP duplicate names",
+			cfg: &CaseConfig{
+				ID:    "test-case",
+				Input: Input{Prompt: "Say hello"},
+				MCP: MCPConfig{Servers: []MCPServer{
+					{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/open.yaml"},
+					{Name: "svc", Mode: "mocked", ConfigRef: "evals/fixtures/mcp/closed.yaml"},
 				}},
 			},
 			wantErr: true,