Merge pull request #10 from mclenhard/feat/add-evals

adds eval testing

Author: ihor-sokoliuk

README.md

@@ -157,6 +157,13 @@ Add this to your `claude_desktop_config.json`:
 }
 ```
 
+## Running evals
+
+The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can see the full documentation [here](https://www.mcpevals.io/docs).
+
+```bash
+SEARXNG_URL=SEARXNG_URL OPENAI_API_KEY=your-key npx mcp-eval evals.ts index.ts
+```
 
 ## License
 

evals.ts

@@ -0,0 +1,23 @@
+//evals.ts
+
+import { EvalConfig } from 'mcp-evals';
+import { openai } from "@ai-sdk/openai";
+import { grade, EvalFunction } from "mcp-evals";
+
+const searxng_web_searchEval: EvalFunction = {
+    name: "searxng_web_search Tool Evaluation",
+    description: "Evaluates searxng_web_search tool functionality",
+    run: async () => {
+        const result = await grade(openai("gpt-4o"), "Search for the latest news on climate change using the searxng_web_search tool and summarize the top findings.");
+        return JSON.parse(result);
+    }
+};
+
+const config: EvalConfig = {
+    model: openai("gpt-4o"),
+    evals: [searxng_web_searchEval ]
+};
+  
+export default config;
+  
+export const evals = [searxng_web_searchEval ];

package.json

@@ -40,6 +40,8 @@
         "postversion": "node scripts/update-version.js && git add index.ts && git commit --amend --no-edit"
     },
     "dependencies": {
+        "@modelcontextprotocol/sdk": "1.10.1",
+        "mcp-evals": "^1.0.18",
         "@modelcontextprotocol/sdk": "1.15.0",
         "node-html-markdown": "^1.3.0"
     },