We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 67fc5d6 commit 8dbd727Copy full SHA for 8dbd727
eval/lib/graders/mcp-tools.ts
@@ -283,8 +283,9 @@ export async function gradeMcpTools(trialArgs: TrialArgs): Promise<McpToolsSumma
283
taskConfig.expectedMcpTools,
284
);
285
286
- // Only return if there were any MCP tool calls
287
- if (mcpToolsSummary.totalCalls > 0) {
+ // Return if there were any MCP tool calls, or if expectations were configured
+ // (so a run that ignores MCP entirely scores 0 rather than undefined)
288
+ if (mcpToolsSummary.totalCalls > 0 || mcpToolsSummary.expectedToolCount) {
289
return mcpToolsSummary;
290
}
291
0 commit comments