diff --git a/AWESOME_WEBMCP.md b/AWESOME_WEBMCP.md index 2bfe3d0..8c56938 100644 --- a/AWESOME_WEBMCP.md +++ b/AWESOME_WEBMCP.md @@ -17,6 +17,8 @@ A curated list of awesome WebMCP demos. - **Example Prompt:** "I want to book a flight from New York to Los Angeles for two people on next Thursday." - [Animal Viewer](https://65s6dw.csb.app/) - A simple codesandbox demo page that shows either a dog or a cat image. - **Example Prompt:** "Show me a dog on this page" +- **Moving Beyond Screen Scraping**: A hands-on example of using WebMCP to create an agentic first experience with 10x fewer tokens + - [Article](https://lnkd.in/daPRAtMX) | [Code](https://lnkd.in/dkZ3Jizn) ## Contributing diff --git a/evals-cli/README.md b/evals-cli/README.md index 589b8f9..398c2d4 100644 --- a/evals-cli/README.md +++ b/evals-cli/README.md @@ -13,9 +13,11 @@ A TypeScript framework for evaluating the tool-calling capabilities of Large Lan The project is structured as follows: - `src/`: Source code. - - `bin/runevals.ts`: Main entry point that sets up the backend and runs the evaluation loop. - - `backend/`: Implementation of LLM backends (e.g., `googleai.ts`, `ollama.ts`). - - `types/`: TypeScript definitions for tools, messages, and evaluations. + - `bin/runevals.ts`: Entry point that loads tool schemas from a JSON file and runs the evaluation loop. + - `bin/webmcpevals.ts`: Entry point that loads tool schemas live from a browser page via the WebMCP API. + - `backend/`: Implementation of LLM backends (e.g., `googleai.ts`, `ollama.ts`). + - `browser/`: Browser automation for WebMCP tool discovery (`webmcp.ts`). + - `types/`: TypeScript definitions for tools, messages, and evaluations. - `examples/`: Detailed examples and test data. - `travel/`: A travel agent example containing `schema.json` and `evals.json`. @@ -23,6 +25,7 @@ The project is structured as follows: - Node.js (v18+ recommended) - A Google AI Studio API Key (for Gemini models) +- Chrome Canary 146+ with the `#enable-webmcp-testing` flag enabled (for `webmcpevals` only) ## Setup @@ -51,46 +54,70 @@ The project is structured as follows: ## Usage -### Running the Travel Example +### `runevals` — file-based tool schemas + +Loads tool schemas from a local JSON file. ```bash node dist/bin/runevals.js --model=gemini-2.5-flash --tools=examples/travel/schema.json --evals=examples/travel/evals.json ``` -### Running evals with Ollama +With Ollama: ```bash node dist/bin/runevals.js --model=qwen3:8b --backend=ollama --tools=examples/travel/schema.json --evals=examples/travel/evals.json ``` +| Argument | Required | Default | Description | +| ----------- | -------- | ------------------ | ------------------------------------- | +| `--tools` | Yes | — | Path to the tool schema JSON file | +| `--evals` | Yes | — | Path to the evals JSON file | +| `--backend` | No | `gemini` | Backend to use (`gemini` or `ollama`) | +| `--model` | No | `gemini-2.5-flash` | Model name | + +### `webmcpevals` — live tool schemas via WebMCP + +Launches Chrome Canary, navigates to the given URL, and retrieves tool schemas live from the page via `navigator.modelContextTesting.listTools()`. Requires Chrome Canary 146+ with the `chrome://flags/#enable-webmcp-testing` flag enabled. + +```bash +node dist/bin/webmcpevals.js --url=https://example.com/my-webmcp-app --evals=examples/travel/evals.json +``` + +| Argument | Required | Default | Description | +| ----------- | -------- | ------------------ | ------------------------------------- | +| `--url` | Yes | — | URL of the page exposing WebMCP tools | +| `--evals` | Yes | — | Path to the evals JSON file | +| `--backend` | No | `gemini` | Backend to use (`gemini` or `ollama`) | +| `--model` | No | `gemini-2.5-flash` | Model name | + ## Argument Constraints You can use constraint operators to match argument values flexibly. A constraint object is identified when **all** its keys start with `$`. ### Supported Operators -| Operator | Description | Example | -|---|---|---| -| **`$pattern`** | Regex match | `{"$pattern": "^2026-\\d{2}$"}` | -| **`$contains`** | Substring match | `{"$contains": "York"}` | -| **`$gt`**, **`$gte`** | Greater than (or equal) | `{"$gte": 1}` | -| **`$lt`**, **`$lte`** | Less than (or equal) | `{"$lt": 100}` | -| **`$type`** | Type check | `{"$type": "string"}` | -| **`$any`** | Presence check | `{"$any": true}` | +| Operator | Description | Example | +| --------------------- | ----------------------- | ------------------------------- | +| **`$pattern`** | Regex match | `{"$pattern": "^2026-\\d{2}$"}` | +| **`$contains`** | Substring match | `{"$contains": "York"}` | +| **`$gt`**, **`$gte`** | Greater than (or equal) | `{"$gte": 1}` | +| **`$lt`**, **`$lte`** | Less than (or equal) | `{"$lt": 100}` | +| **`$type`** | Type check | `{"$type": "string"}` | +| **`$any`** | Presence check | `{"$any": true}` | ### Example ```json { - "expectedCall": { - "functionName": "searchFlights", - "arguments": { - "destination": "NYC", - "outboundDate": { "$pattern": "^2026-01-\\d{2}$" }, - "passengers": { "$gte": 1 }, - "preferences": { "$any": true } - } + "expectedCall": { + "functionName": "searchFlights", + "arguments": { + "destination": "NYC", + "outboundDate": { "$pattern": "^2026-01-\\d{2}$" }, + "passengers": { "$gte": 1 }, + "preferences": { "$any": true } } + } } ``` diff --git a/evals-cli/package-lock.json b/evals-cli/package-lock.json index a13652e..1933f00 100644 --- a/evals-cli/package-lock.json +++ b/evals-cli/package-lock.json @@ -15,7 +15,8 @@ "cli-progress": "^3.12.0", "dotenv": "^17.2.3", "minimist": "^1.2.8", - "ollama": "^0.6.3" + "ollama": "^0.6.3", + "puppeteer-core": "^24.37.5" }, "devDependencies": { "@types/node": "^25.0.10", @@ -158,6 +159,33 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@puppeteer/browsers": { + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", + "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.4", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@types/cli-progress": { "version": "3.11.6", "resolved": "https://registry.npmjs.org/@types/cli-progress/-/cli-progress-3.11.6.tgz", @@ -182,6 +210,16 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -215,12 +253,130 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/b4a": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.0.tgz", + "integrity": "sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.8.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", + "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.4.tgz", + "integrity": "sha512-POK4oplfA7P7gqvetNmCs4CNtm9fNsx+IAh7jH7GgU0OJdge2rso0R20TNWVq6VoWcCvsTdlNDaleLHGaKx8CA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.8.0.tgz", + "integrity": "sha512-reUN0M2sHRqCdG4lUK3Fw8w98eeUIZHL5c3H7Mbhk2yVBL+oofgaIp0ieLfD5QXwPCypBpmEEKU2WZKzbAk8GA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz", + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-path": "^3.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -241,6 +397,15 @@ ], "license": "MIT" }, + "node_modules/basic-ftp": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.1.0.tgz", + "integrity": "sha512-RkaJzeJKDbaDWTIPiJwubyljaEPwpVWkm9Rt5h9Nd6h7tEXTJ3VB4qxdZBioV7JO5yLUaOKwz7vDOzlncUsegw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bignumber.js": { "version": "9.3.1", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", @@ -259,12 +424,34 @@ "balanced-match": "^1.0.0" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/cli-progress": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", @@ -277,6 +464,73 @@ "node": ">=4" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -335,6 +589,26 @@ } } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1566079", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1566079.tgz", + "integrity": "sha512-MJfAEA1UfVhSs7fbSQOG4czavUp1ajfg6prlAN0+cmfa2zNjaIbvq8VneP7do1WAQQIvgNJWSMeP6UyI90gIlQ==", + "license": "BSD-3-Clause" + }, "node_modules/dotenv": { "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", @@ -368,12 +642,126 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -454,6 +842,53 @@ "node": ">=18" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/glob": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", @@ -514,6 +949,19 @@ "node": ">=18" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -527,6 +975,15 @@ "node": ">= 14" } }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -632,12 +1089,27 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -685,6 +1157,47 @@ "whatwg-fetch": "^3.6.20" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -716,6 +1229,21 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/protobufjs": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", @@ -740,6 +1268,77 @@ "node": ">=12.0.0" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer-core": { + "version": "24.37.5", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.37.5.tgz", + "integrity": "sha512-ybL7iE78YPN4T6J+sPLO7r0lSByp/0NN6PvfBEql219cOnttoTFzCWKiBOjstXSqi/OKpwae623DWAsL7cn2MQ==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.0", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1566079", + "typed-query-selector": "^2.12.0", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.19.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/rimraf": { "version": "5.0.10", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", @@ -775,6 +1374,18 @@ ], "license": "MIT" }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -808,6 +1419,65 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/streamx": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -916,6 +1586,62 @@ "node": ">=8" } }, + "node_modules/tar-fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "license": "MIT", + "optional": true, + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -945,6 +1671,12 @@ "node": ">= 8" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "license": "Apache-2.0" + }, "node_modules/whatwg-fetch": { "version": "3.6.20", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", @@ -1060,6 +1792,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, "node_modules/ws": { "version": "8.19.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", @@ -1080,6 +1818,61 @@ "optional": true } } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/evals-cli/package.json b/evals-cli/package.json index 3ac2cc5..be5f322 100644 --- a/evals-cli/package.json +++ b/evals-cli/package.json @@ -19,6 +19,7 @@ "cli-progress": "^3.12.0", "dotenv": "^17.2.3", "minimist": "^1.2.8", - "ollama": "^0.6.3" + "ollama": "^0.6.3", + "puppeteer-core": "^24.37.5" } } diff --git a/evals-cli/src/bin/webmcpevals.ts b/evals-cli/src/bin/webmcpevals.ts new file mode 100644 index 0000000..391f22e --- /dev/null +++ b/evals-cli/src/bin/webmcpevals.ts @@ -0,0 +1,116 @@ +/** + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { GoogleAiBackend } from "../backend/googleai.js"; +import { readFile, writeFile } from "fs/promises"; +import * as dotenv from "dotenv"; +import { functionCallOutcome } from "../utils.js"; +import { Eval, TestResult } from "../types/evals.js"; +import { OllamaBackend } from "../backend/ollama.js"; +import { SingleBar } from "cli-progress"; +import minimist from "minimist"; +import { WebmcpConfig } from "../types/config.js"; +import { renderWebmcpReport } from "../report/report.js"; +import { listToolsFromPage } from "../browser/webmcp.js"; + +const SYSTEM_PROMPT = ` +# INSTRUCTIONS +You are an agent helping a user navigate a page via the tools made available to you. You must +use the tools available to help the user. + +# ADDITIONAL CONTEXT +Today's date is: Monday 19th of January, 2026. +`; + +dotenv.config(); + +const args = minimist(process.argv.slice(2)); + +if (!args.url) { + console.error("The 'url' argument is required."); + process.exit(1); +} + +if (!args.evals) { + console.error("The 'evals' argument is required."); + process.exit(1); +} + +if (args.backend && args.backend === "ollama" && !args.model) { + console.error( + "The 'model' argument is required when 'backend' is set to 'ollama'.", + ); + process.exit(1); +} + +const config: WebmcpConfig = { + url: args.url, + evalsFile: args.evals, + backend: args.backend || "gemini", + model: args.model || "gemini-2.5-flash", +}; + +const tools = await listToolsFromPage(config.url); + +const tests: Array = JSON.parse( + await readFile(config.evalsFile, "utf-8"), +); + +let backend; +switch (config.backend) { + case "ollama": + backend = new OllamaBackend( + process.env.OLLAMA_HOST!, + config.model, + SYSTEM_PROMPT, + tools, + ); + break; + default: + backend = new GoogleAiBackend( + process.env.GOOGLE_AI!, + config.model, + SYSTEM_PROMPT, + tools, + ); +} + +const progressBar = new SingleBar({ + format: + "progress [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | accuracy: {accuracy}%", +}); +progressBar.start(tests.length, 0, { accuracy: "0.00" }); +let testCount = 0; +let passCount = 0; +let failCount = 0; +let errorCount = 0; +const testResults: Array = []; +for (const test of tests) { + testCount++; + try { + const response = await backend.execute(test.messages); + const outcome = functionCallOutcome(test.expectedCall, response); + testResults.push({ test, response, outcome }); + outcome === "pass" ? passCount++ : failCount++; + } catch (e) { + console.warn("Error running test:", e); + errorCount++; + } + progressBar.update(testCount, { + accuracy: ((passCount / testCount) * 100).toFixed(2), + }); +} + +const report = renderWebmcpReport(config, { + results: testResults, + testCount, + errorCount, + failCount, + passCount, +}); + +await writeFile("report.html", report); +console.log("\nReport saved to report.html"); +process.exit(); diff --git a/evals-cli/src/browser/webmcp.ts b/evals-cli/src/browser/webmcp.ts new file mode 100644 index 0000000..67847ed --- /dev/null +++ b/evals-cli/src/browser/webmcp.ts @@ -0,0 +1,146 @@ +/** + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import puppeteer, { Browser } from "puppeteer-core"; +import * as os from "os"; +import * as path from "path"; +import * as fs from "fs"; +import { Tool } from "../types/tools.js"; + +type WebMcpToolSchema = { + name: string; + description: string; + inputSchema: object | null; +}; + +const CHROME_CANARY_PATHS: string[] = [ + // Windows + path.join( + os.homedir(), + "AppData", + "Local", + "Google", + "Chrome SxS", + "Application", + "chrome.exe", + ), + // macOS + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + // Linux unstable channel + "/usr/bin/google-chrome-unstable", + "/opt/google/chrome-unstable/google-chrome", +]; + +function findChromePath(): string { + for (const candidate of CHROME_CANARY_PATHS) { + if (fs.existsSync(candidate)) { + return candidate; + } + } + throw new Error( + "Chrome Canary not found. Please install Chrome Canary (version 146+).\n" + + "Checked paths:\n" + + CHROME_CANARY_PATHS.map((p) => ` - ${p}`).join("\n"), + ); +} + +function mapToTools(rawTools: WebMcpToolSchema[]): Tool[] { + return rawTools.map((t) => { + const schema = t.inputSchema; + const parameters = + typeof schema === "string" ? JSON.parse(schema) : (schema ?? {}); + return { + description: t.description, + functionName: t.name, + parameters, + }; + }); +} + +/** + * Launches Chrome Canary, navigates to the given URL, and retrieves the list + * of tools exposed by the page via the WebMCP API + * (`navigator.modelContextTesting.listTools()`). + * + * Requires Chrome Canary 146+ with the `chrome://flags/#enable-webmcp-testing` + * flag enabled. The browser is always closed after the tools are retrieved, + * even if an error occurs. + * + * @param url - The URL of the page to load. The page must implement the WebMCP + * API and expose at least one tool. + * @returns A promise that resolves to the list of tools exposed by the page. + * @throws If Chrome Canary is not found, the page fails to load, the WebMCP + * API is unavailable, or no tools are returned. + */ +export async function listToolsFromPage(url: string): Promise { + const executablePath = findChromePath(); + let browser: Browser | null = null; + + try { + console.log(`Launching Chrome Canary from: ${executablePath}`); + browser = await puppeteer.launch({ + executablePath, + headless: true, + args: [ + "--enable-features=WebMCPTesting", + "--no-sandbox", + "--disable-setuid-sandbox", + ], + }); + + const page = await browser.newPage(); + + console.log(`Navigating to: ${url}`); + const response = await page.goto(url, { + waitUntil: "networkidle2", + timeout: 30000, + }); + + if (!response || !response.ok()) { + throw new Error( + `Failed to navigate to ${url}. HTTP status: ${response?.status() ?? "unknown"}`, + ); + } + + const rawTools = await page.evaluate(async () => { + if ( + typeof (navigator as unknown as Record)[ + "modelContextTesting" + ] === "undefined" + ) { + return null; + } + return await ( + navigator as unknown as { + modelContextTesting: { listTools: () => Promise }; + } + ).modelContextTesting.listTools(); + }); + + if (rawTools === null) { + throw new Error( + "The WebMCP API (window.navigator.modelContextTesting) is not available on this page.\n" + + "Please ensure:\n" + + " 1. You are using Chrome Canary version 146 or later.\n" + + " 2. The flag chrome://flags/#enable-webmcp-testing is enabled.\n" + + ` 3. The page at ${url} implements the WebMCP API.`, + ); + } + + if (!Array.isArray(rawTools) || rawTools.length === 0) { + throw new Error( + `The WebMCP API returned no tools from ${url}. ` + + "Ensure the page exposes tools via modelContextTesting.listTools().", + ); + } + + console.log(`Found ${rawTools.length} tool(s) via WebMCP API.`); + return mapToTools(rawTools as WebMcpToolSchema[]); + } finally { + if (browser) { + await browser.close(); + } + } +} diff --git a/evals-cli/src/matcher.ts b/evals-cli/src/matcher.ts index 0109a88..a29ec8e 100644 --- a/evals-cli/src/matcher.ts +++ b/evals-cli/src/matcher.ts @@ -5,7 +5,7 @@ /** * Checks if the actual argument matches the expected argument, supporting both exact matching and constraints. - * + * * If the expected argument is a constraint object (all keys start with `$`), it evaluates the constraints. * Otherwise, it performs a recursive deep equality check, allowing nested constraints. * diff --git a/evals-cli/src/report/report.ts b/evals-cli/src/report/report.ts index e343405..badbcfb 100644 --- a/evals-cli/src/report/report.ts +++ b/evals-cli/src/report/report.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { Config } from "../types/config.js"; +import { Config, WebmcpConfig } from "../types/config.js"; import { Message, TestResult, TestResults } from "../types/evals.js"; import { matchesArgument } from "../matcher.js"; @@ -174,3 +174,59 @@ function renderMessage(message: Message): string {
${content}
`; } + +export function renderWebmcpReport( + config: WebmcpConfig, + testResults: TestResults, +): string { + return ` + + + + + + + + +

Eval Results

+
+

Test configuration

+ ${renderWebmcpConfiguration(config)} +
+
+

Results

+

Summary

+
+ ${renderEvalsSummary(testResults)} +
+
+

Details

+ ${renderDetails(testResults.results)} +
+
+ +`; +} + +function renderWebmcpConfiguration(config: WebmcpConfig): string { + return ` +
    +
  • URL: ${config.url}
    • +
  • Evals: ${config.evalsFile}
    • +
  • Backend: ${config.backend}
    • +
  • Model: ${config.model}
    • +
`; +} diff --git a/evals-cli/src/test/matcher.test.ts b/evals-cli/src/test/matcher.test.ts index bd31e22..5ffbd8f 100644 --- a/evals-cli/src/test/matcher.test.ts +++ b/evals-cli/src/test/matcher.test.ts @@ -8,172 +8,172 @@ import * as assert from "node:assert"; import { matchesArgument } from "../matcher.js"; describe("matcher", () => { - describe("exact matching", () => { - it("matches primitive values", () => { - assert.strictEqual(matchesArgument(1, 1), true); - assert.strictEqual(matchesArgument("hello", "hello"), true); - assert.strictEqual(matchesArgument(true, true), true); - assert.strictEqual(matchesArgument(null, null), true); - - assert.strictEqual(matchesArgument(1, 2), false); - assert.strictEqual(matchesArgument("hello", "world"), false); - assert.strictEqual(matchesArgument(true, false), false); - assert.strictEqual(matchesArgument(null, undefined), false); - }); - - it("matches objects deeply", () => { - assert.strictEqual(matchesArgument({ a: 1 }, { a: 1 }), true); - assert.strictEqual( - matchesArgument({ a: { b: 2 } }, { a: { b: 2 } }), - true, - ); - - assert.strictEqual(matchesArgument({ a: 1 }, { a: 2 }), false); - assert.strictEqual(matchesArgument({ a: 1 }, { b: 1 }), false); - assert.strictEqual( - matchesArgument({ a: { b: 2 } }, { a: { b: 3 } }), - false, - ); - }); - - it("matches arrays deeply", () => { - assert.strictEqual(matchesArgument([1, 2], [1, 2]), true); - assert.strictEqual(matchesArgument([1, [2]], [1, [2]]), true); - - assert.strictEqual(matchesArgument([1, 2], [1, 3]), false); - assert.strictEqual(matchesArgument([1, 2], [1, 2, 3]), false); - }); + describe("exact matching", () => { + it("matches primitive values", () => { + assert.strictEqual(matchesArgument(1, 1), true); + assert.strictEqual(matchesArgument("hello", "hello"), true); + assert.strictEqual(matchesArgument(true, true), true); + assert.strictEqual(matchesArgument(null, null), true); + + assert.strictEqual(matchesArgument(1, 2), false); + assert.strictEqual(matchesArgument("hello", "world"), false); + assert.strictEqual(matchesArgument(true, false), false); + assert.strictEqual(matchesArgument(null, undefined), false); }); - describe("constraints", () => { - describe("$pattern", () => { - it("matches strings against regex", () => { - assert.strictEqual( - matchesArgument({ $pattern: "^2026-\\d{2}$" }, "2026-01"), - true, - ); - assert.strictEqual( - matchesArgument({ $pattern: "foo" }, "foobar"), - true, - ); - - assert.strictEqual( - matchesArgument({ $pattern: "^2026-\\d{2}$" }, "2025-01"), - false, - ); - assert.strictEqual( - matchesArgument({ $pattern: "^foo$" }, "foobar"), - false, - ); - }); - - it("fails if actual is not a string", () => { - assert.strictEqual(matchesArgument({ $pattern: ".*" }, 123), false); - assert.strictEqual(matchesArgument({ $pattern: ".*" }, null), false); - }); - }); - - describe("$contains", () => { - it("matches strings containing substring", () => { - assert.strictEqual( - matchesArgument({ $contains: "bar" }, "foobar"), - true, - ); - assert.strictEqual(matchesArgument({ $contains: "foo" }, "foo"), true); - - assert.strictEqual( - matchesArgument({ $contains: "baz" }, "foobar"), - false, - ); - }); - - it("fails if actual is not a string", () => { - assert.strictEqual(matchesArgument({ $contains: "foo" }, 123), false); - assert.strictEqual(matchesArgument({ $contains: "foo" }, null), false); - }); - }); - - describe("numeric comparisons", () => { - it("$gt", () => { - assert.strictEqual(matchesArgument({ $gt: 10 }, 11), true); - assert.strictEqual(matchesArgument({ $gt: 10 }, 10), false); - }); - it("$gte", () => { - assert.strictEqual(matchesArgument({ $gte: 10 }, 10), true); - assert.strictEqual(matchesArgument({ $gte: 10 }, 9), false); - }); - it("$lt", () => { - assert.strictEqual(matchesArgument({ $lt: 10 }, 9), true); - assert.strictEqual(matchesArgument({ $lt: 10 }, 10), false); - }); - it("$lte", () => { - assert.strictEqual(matchesArgument({ $lte: 10 }, 10), true); - assert.strictEqual(matchesArgument({ $lte: 10 }, 11), false); - }); - it("fails if actual is not a number", () => { - assert.strictEqual(matchesArgument({ $gt: 10 }, "11"), false); - assert.strictEqual(matchesArgument({ $gt: 10 }, null), false); - }); - }); - - describe("$type", () => { - it("matches specific types", () => { - assert.strictEqual(matchesArgument({ $type: "string" }, "foo"), true); - assert.strictEqual(matchesArgument({ $type: "number" }, 123), true); - assert.strictEqual(matchesArgument({ $type: "boolean" }, true), true); - assert.strictEqual(matchesArgument({ $type: "object" }, {}), true); - assert.strictEqual(matchesArgument({ $type: "array" }, []), true); - assert.strictEqual(matchesArgument({ $type: "null" }, null), true); - - assert.strictEqual(matchesArgument({ $type: "string" }, 123), false); - assert.strictEqual(matchesArgument({ $type: "number" }, "123"), false); - assert.strictEqual(matchesArgument({ $type: "array" }, {}), false); - assert.strictEqual(matchesArgument({ $type: "object" }, []), false); - }); - }); - - describe("$any", () => { - it("matches anything", () => { - assert.strictEqual(matchesArgument({ $any: true }, "foo"), true); - assert.strictEqual(matchesArgument({ $any: true }, null), true); - assert.strictEqual(matchesArgument({ $any: true }, undefined), true); // undefined usually shouldn't happen in JSON arguments but good to check - }); - }); - - describe("recursive constraints", () => { - it("matches nested constraints", () => { - const schema = { - a: { $gt: 10 }, - b: { c: { $contains: "hello" } }, - }; - assert.strictEqual( - matchesArgument(schema, { a: 11, b: { c: "hello world" } }), - true, - ); - assert.strictEqual( - matchesArgument(schema, { a: 10, b: { c: "hello world" } }), - false, - ); - assert.strictEqual( - matchesArgument(schema, { a: 11, b: { c: "bye world" } }), - false, - ); - }); - - it("matches array elements with constraints", () => { - const schema = { - list: [{ $gt: 10 }, { $type: "string" }], - }; - assert.strictEqual( - matchesArgument(schema, { list: [11, "foo"] }), - true, - ); - assert.strictEqual( - matchesArgument(schema, { list: [10, "foo"] }), - false, - ); - assert.strictEqual(matchesArgument(schema, { list: [11, 123] }), false); - }); - }); + it("matches objects deeply", () => { + assert.strictEqual(matchesArgument({ a: 1 }, { a: 1 }), true); + assert.strictEqual( + matchesArgument({ a: { b: 2 } }, { a: { b: 2 } }), + true, + ); + + assert.strictEqual(matchesArgument({ a: 1 }, { a: 2 }), false); + assert.strictEqual(matchesArgument({ a: 1 }, { b: 1 }), false); + assert.strictEqual( + matchesArgument({ a: { b: 2 } }, { a: { b: 3 } }), + false, + ); }); + + it("matches arrays deeply", () => { + assert.strictEqual(matchesArgument([1, 2], [1, 2]), true); + assert.strictEqual(matchesArgument([1, [2]], [1, [2]]), true); + + assert.strictEqual(matchesArgument([1, 2], [1, 3]), false); + assert.strictEqual(matchesArgument([1, 2], [1, 2, 3]), false); + }); + }); + + describe("constraints", () => { + describe("$pattern", () => { + it("matches strings against regex", () => { + assert.strictEqual( + matchesArgument({ $pattern: "^2026-\\d{2}$" }, "2026-01"), + true, + ); + assert.strictEqual( + matchesArgument({ $pattern: "foo" }, "foobar"), + true, + ); + + assert.strictEqual( + matchesArgument({ $pattern: "^2026-\\d{2}$" }, "2025-01"), + false, + ); + assert.strictEqual( + matchesArgument({ $pattern: "^foo$" }, "foobar"), + false, + ); + }); + + it("fails if actual is not a string", () => { + assert.strictEqual(matchesArgument({ $pattern: ".*" }, 123), false); + assert.strictEqual(matchesArgument({ $pattern: ".*" }, null), false); + }); + }); + + describe("$contains", () => { + it("matches strings containing substring", () => { + assert.strictEqual( + matchesArgument({ $contains: "bar" }, "foobar"), + true, + ); + assert.strictEqual(matchesArgument({ $contains: "foo" }, "foo"), true); + + assert.strictEqual( + matchesArgument({ $contains: "baz" }, "foobar"), + false, + ); + }); + + it("fails if actual is not a string", () => { + assert.strictEqual(matchesArgument({ $contains: "foo" }, 123), false); + assert.strictEqual(matchesArgument({ $contains: "foo" }, null), false); + }); + }); + + describe("numeric comparisons", () => { + it("$gt", () => { + assert.strictEqual(matchesArgument({ $gt: 10 }, 11), true); + assert.strictEqual(matchesArgument({ $gt: 10 }, 10), false); + }); + it("$gte", () => { + assert.strictEqual(matchesArgument({ $gte: 10 }, 10), true); + assert.strictEqual(matchesArgument({ $gte: 10 }, 9), false); + }); + it("$lt", () => { + assert.strictEqual(matchesArgument({ $lt: 10 }, 9), true); + assert.strictEqual(matchesArgument({ $lt: 10 }, 10), false); + }); + it("$lte", () => { + assert.strictEqual(matchesArgument({ $lte: 10 }, 10), true); + assert.strictEqual(matchesArgument({ $lte: 10 }, 11), false); + }); + it("fails if actual is not a number", () => { + assert.strictEqual(matchesArgument({ $gt: 10 }, "11"), false); + assert.strictEqual(matchesArgument({ $gt: 10 }, null), false); + }); + }); + + describe("$type", () => { + it("matches specific types", () => { + assert.strictEqual(matchesArgument({ $type: "string" }, "foo"), true); + assert.strictEqual(matchesArgument({ $type: "number" }, 123), true); + assert.strictEqual(matchesArgument({ $type: "boolean" }, true), true); + assert.strictEqual(matchesArgument({ $type: "object" }, {}), true); + assert.strictEqual(matchesArgument({ $type: "array" }, []), true); + assert.strictEqual(matchesArgument({ $type: "null" }, null), true); + + assert.strictEqual(matchesArgument({ $type: "string" }, 123), false); + assert.strictEqual(matchesArgument({ $type: "number" }, "123"), false); + assert.strictEqual(matchesArgument({ $type: "array" }, {}), false); + assert.strictEqual(matchesArgument({ $type: "object" }, []), false); + }); + }); + + describe("$any", () => { + it("matches anything", () => { + assert.strictEqual(matchesArgument({ $any: true }, "foo"), true); + assert.strictEqual(matchesArgument({ $any: true }, null), true); + assert.strictEqual(matchesArgument({ $any: true }, undefined), true); // undefined usually shouldn't happen in JSON arguments but good to check + }); + }); + + describe("recursive constraints", () => { + it("matches nested constraints", () => { + const schema = { + a: { $gt: 10 }, + b: { c: { $contains: "hello" } }, + }; + assert.strictEqual( + matchesArgument(schema, { a: 11, b: { c: "hello world" } }), + true, + ); + assert.strictEqual( + matchesArgument(schema, { a: 10, b: { c: "hello world" } }), + false, + ); + assert.strictEqual( + matchesArgument(schema, { a: 11, b: { c: "bye world" } }), + false, + ); + }); + + it("matches array elements with constraints", () => { + const schema = { + list: [{ $gt: 10 }, { $type: "string" }], + }; + assert.strictEqual( + matchesArgument(schema, { list: [11, "foo"] }), + true, + ); + assert.strictEqual( + matchesArgument(schema, { list: [10, "foo"] }), + false, + ); + assert.strictEqual(matchesArgument(schema, { list: [11, 123] }), false); + }); + }); + }); }); diff --git a/evals-cli/src/types/config.ts b/evals-cli/src/types/config.ts index c6fb2e0..6c82824 100644 --- a/evals-cli/src/types/config.ts +++ b/evals-cli/src/types/config.ts @@ -9,3 +9,10 @@ export type Config = { backend: string; model: string; }; + +export type WebmcpConfig = { + url: string; + evalsFile: string; + backend: string; + model: string; +};