diff --git a/README.md b/README.md index 6141dfd..30df0ba 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,20 @@ four-skill candidate archive, and contract, package, target, and eval receipts under ignored `dist/`. `bun run verify:artifacts` performs clean tarball installs and runtime checks. Nothing here publishes, releases, deploys, submits, or calls production during pull-request CI. + +Live eval runners are `bun run eval:responses`, `eval:codex`, `eval:openrouter`, +`eval:claude`, and `eval:omp`. Hermetic replay is `eval:replay`. Every live runner +requires `ASK_GINA_ACCESS_TOKEN`. Responses and Codex also need `OPENAI_API_KEY`; +Codex adds `CODEX_EVAL_EXECUTABLE` and `CODEX_EVAL_EXECUTABLE_SHA256`; OpenRouter +needs `OPENROUTER_API_KEY`; Claude needs `ANTHROPIC_API_KEY` and +`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_API_KEY`, +`OMP_EVAL_EXECUTABLE`, `OMP_EVAL_EXECUTABLE_SHA256`, a local Docker engine, and +`--provider openai|anthropic|openrouter`. Optional `--max-steps` applies only to +OpenRouter and `--max-turns` only to Claude. Both default to 8 and accept 1 to 32. +OMP has no step or turn flag. Native Codex, Claude, and OMP paths use explicit API +keys, not a saved personal login. OpenRouter uses local AI SDK MCP; Responses uses +OpenAI-hosted MCP. Neither proves native plugin activation. Codex and Claude +adapters distinguish native skill events from task conformance. Claude's live +plugin activation remains unverified; offline fixtures and OMP Docker/runtime +proof are not measured native-agent evidence. Flags, capture, and publication +rules are in `packages/evals/README.md`. diff --git a/apps/evals/__tests__/public-results.test.ts b/apps/evals/__tests__/public-results.test.ts index c0f215f..58b6c28 100644 --- a/apps/evals/__tests__/public-results.test.ts +++ b/apps/evals/__tests__/public-results.test.ts @@ -97,6 +97,88 @@ describe("public artifact browser boundary", () => { }); }); + it("accepts nested OpenRouter model identities in publications and current-index summaries", () => { + const model = "openrouter/openai/gpt-5.1"; + const current = publication(currentBytes); + if (current.content.kind !== "result") throw new Error("Expected a result publication fixture"); + const entry = history.publications[0]; + if (entry?.summary === null || entry?.summary === undefined) { + throw new Error("Expected a current index summary fixture"); + } + + expect( + parsePublicArtifact( + encode({ + ...current, + content: { + ...current.content, + result: { + ...current.content.result, + configuration: { ...current.content.result.configuration, model }, + }, + }, + }), + ).kind, + ).toBe("publication"); + expect( + parsePublicArtifact( + encode({ + ...history, + publications: [ + { ...entry, summary: { ...entry.summary, model } }, + ...history.publications.slice(1), + ], + }), + ).kind, + ).toBe("index"); + }); + + it("rejects malformed model identities without relaxing generic identifiers", () => { + const current = publication(currentBytes); + if (current.content.kind !== "result") throw new Error("Expected a result publication fixture"); + const entry = history.publications[0]; + if (entry?.summary === null || entry?.summary === undefined) { + throw new Error("Expected a current index summary fixture"); + } + + const malformedModels = [ + "openrouter//gpt-5.1", + "openrouter/../gpt-5.1", + `openrouter/${"a".repeat(118)}`, + ] as const; + for (const model of malformedModels) { + expect( + parsePublicArtifact( + encode({ + ...current, + content: { + ...current.content, + result: { + ...current.content.result, + configuration: { ...current.content.result.configuration, model }, + }, + }, + }), + ).kind, + ).toBe("unsupported"); + expect( + parsePublicArtifact( + encode({ + ...history, + publications: [ + { ...entry, summary: { ...entry.summary, model } }, + ...history.publications.slice(1), + ], + }), + ).kind, + ).toBe("unsupported"); + } + + expect(parsePublicArtifact(encode({ ...current, publicationId: "openai/gpt-5.1" })).kind).toBe( + "unsupported", + ); + }); + it("rejects malformed UTF-8 and BOM-prefixed JSON rather than normalizing the input", () => { expect( parsePublicArtifact(new Uint8Array([0x7b, 0x22, 0xc0, 0xaf, 0x22, 0x7d]).buffer).kind, diff --git a/apps/evals/src/lib/public-results.ts b/apps/evals/src/lib/public-results.ts index 0a0b919..1f4e810 100644 --- a/apps/evals/src/lib/public-results.ts +++ b/apps/evals/src/lib/public-results.ts @@ -10,6 +10,8 @@ export type ParsedPublicArtifact = const WHITESPACE_ONLY = /^\s*$/u; const IDENTIFIER = /^[A-Za-z0-9](?:[A-Za-z0-9._:-]{0,126}[A-Za-z0-9])?$/u; +const MODEL_IDENTIFIER = + /^[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?(?:\/[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?)*$/u; const SHA_256 = /^[a-f0-9]{64}$/u; const ATTEMPT_ID = /^attempt-[a-f0-9]{64}$/u; const UTC_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{3})?Z$/u; @@ -39,6 +41,9 @@ const isText = (value: unknown, max = 128): value is string => const isIdentifier = (value: unknown): value is string => isText(value) && IDENTIFIER.test(value); +const isModelIdentifier = (value: unknown): value is string => + isText(value) && MODEL_IDENTIFIER.test(value); + const isSha256 = (value: unknown): value is string => typeof value === "string" && SHA_256.test(value); @@ -231,7 +236,7 @@ const hasResult = (value: unknown): value is PublicEvalResult => { isObject(configuration) && (configuration.availability === "pinned" || configuration.availability === "labels_only") && isIdentifier(configuration.candidate) && - isIdentifier(configuration.model) && + isModelIdentifier(configuration.model) && (configuration.reasoning === null || isIdentifier(configuration.reasoning)) && (configuration.pinnedSha256 === null || isSha256(configuration.pinnedSha256)) && isObject(coverage) && @@ -386,7 +391,7 @@ const hasIndex = (value: unknown): value is PublicEvalIndex => { (isObject(summary) && isIdentifier(summary.suiteId) && isIdentifier(summary.candidate) && - isIdentifier(summary.model) && + isModelIdentifier(summary.model) && isTimestamp(summary.startedAt))) && Array.isArray(revisions) && revisions.length > 0 diff --git a/bun.lock b/bun.lock index 78fbb15..3d568a1 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,5 @@ { - "lockfileVersion": 2, + "lockfileVersion": 3, "configVersion": 1, "workspaces": { "": { @@ -77,12 +77,24 @@ "name": "@askgina/evals", "version": "0.1.0", "dependencies": { + "@ai-sdk/harness": "1.0.102", + "@ai-sdk/harness-acp": "1.0.40", + "@ai-sdk/mcp": "2.0.45", "@askgina/contracts": "workspace:*", "@askgina/plugin-core": "workspace:*", "@askgina/sdk": "workspace:*", "@effect/platform-bun": "4.0.0-rc.111", + "@openrouter/ai-sdk-provider": "3.0.0", + "ai": "7.0.93", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "dockerode": "4.0.12", "effect": "4.0.0-rc.111", "yaml": "2.8.3", + "zod": "4.1.8", + }, + "devDependencies": { + "@types/dockerode": "4.0.1", }, }, "packages/sdk": { @@ -111,6 +123,9 @@ }, "overrides": { "@effect/platform-node-shared": "4.0.0-rc.111", + "dockerode": { + "uuid": "11.1.1", + }, "esbuild": "0.28.1", "oxlint": "1.78.0", "qs": "6.16.0", @@ -118,6 +133,18 @@ "packages": { "@adobe/css-tools": ["@adobe/css-tools@4.5.0", "", {}, "sha512-6OzddxPio9UiWTCemp4N8cYLV2ZN1ncRnV1cVGtve7dhPOtRkleRyx32GQCYSwDYgaHU3USMm84tNsvKzRCa1Q=="], + "@ai-sdk/gateway": ["@ai-sdk/gateway@4.0.75", "", { "dependencies": { "@ai-sdk/provider": "4.0.10", "@ai-sdk/provider-utils": "5.0.36", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-HOnhw3oXtBnboBF7kAKipjToCN6+5w6EuukiUKiULcxBitS+vbHRRv9IUBcq+Z1wkXcJjfywKrt5r++I6OYyXg=="], + + "@ai-sdk/harness": ["@ai-sdk/harness@1.0.102", "", { "dependencies": { "@ai-sdk/provider": "4.0.10", "@ai-sdk/provider-utils": "5.0.36", "ai": "7.0.93" }, "peerDependencies": { "ws": "^8.21.0", "zod": "^3.25.76 || ^4.1.8" }, "optionalPeers": ["ws"] }, "sha512-DV+TwVPmuiq4V9xuhazlM9rzOftcO1qx1OF46q9bmetknz86fuUcipGbr6esgMIgAA5vJJgf8+2B8/83TIQxjA=="], + + "@ai-sdk/harness-acp": ["@ai-sdk/harness-acp@1.0.40", "", { "dependencies": { "@ai-sdk/harness": "1.0.102", "@ai-sdk/provider-utils": "5.0.36", "ws": "^8.21.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-WLjkLKODyklhXi9ttlNGm9C7RtJ0mlTTeElohWdbc5ovYXWcQoed8C25jNWG+4JRd6fie5gDJBBXfBnotL+mZw=="], + + "@ai-sdk/mcp": ["@ai-sdk/mcp@2.0.45", "", { "dependencies": { "@ai-sdk/provider": "4.0.10", "@ai-sdk/provider-utils": "5.0.36", "cross-spawn": "^7.0.6", "pkce-challenge": "^5.0.1" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ROukI/LfoPHf5F4gi1GKQebK5658fR1EG3kt0P0vzrfSy1FAGQ5FyKr9j2lbkXAV9X3kfpf9LJdif7w67XDtgw=="], + + "@ai-sdk/provider": ["@ai-sdk/provider@4.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-fX2ENAc7iDpZ+Wp4+Rk06Usn/Ys7dI9uAkGv0jlF6XVrW13NkRWx5Ou+U6lIM2E1fTLkCs16GGrUAaVvroag7A=="], + + "@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@5.0.36", "", { "dependencies": { "@ai-sdk/provider": "4.0.10", "@standard-schema/spec": "^1.1.0", "@workflow/serde": "4.1.0", "eventsource-parser": "^3.0.8", "undici": "^7.29.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-MFXBn6XDyf37PNQAge/HTatPJE8Vmg/g/w4WPtjSV53jq8FKAzoaN5+43hsdQa9bqgN+/13jxug9ChvsG+godQ=="], + "@alloc/quick-lru": ["@alloc/quick-lru@5.3.0", "", {}, "sha512-U4+70Pc5ZS9osnCBCE5Jha/ciHM+Yp+CNMNC/7HvYbNRk1Ldd+f7qO65W5qfhu/TCv+/ozljlXXe9Nj8419DMA=="], "@askgina/cli": ["@askgina/cli@workspace:packages/cli"], @@ -164,6 +191,8 @@ "@babel/types": ["@babel/types@7.29.8", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg=="], + "@balena/dockerignore": ["@balena/dockerignore@1.0.2", "", {}, "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="], + "@base-ui/react": ["@base-ui/react@1.5.0", "", { "dependencies": { "@babel/runtime": "^7.29.2", "@base-ui/utils": "0.2.9", "@floating-ui/react-dom": "^2.1.8", "@floating-ui/utils": "^0.2.11", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "@date-fns/tz": "^1.2.0", "@types/react": "^17 || ^18 || ^19", "date-fns": "^4.0.0", "react": "^17 || ^18 || ^19", "react-dom": "^17 || ^18 || ^19" }, "optionalPeers": ["@date-fns/tz", "@types/react", "date-fns"] }, "sha512-z1gSAlced1yY+iM+mHDEtIkD8UI3Ebs52MuBPxvV6f5hRutk+xvCH/wuB7hDqDzK9JG5FoMz5nhrqtSs1wjt1A=="], "@base-ui/utils": ["@base-ui/utils@0.2.9", "", { "dependencies": { "@babel/runtime": "^7.29.2", "@floating-ui/utils": "^0.2.11", "reselect": "^5.1.1", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "@types/react": "^17 || ^18 || ^19", "react": "^17 || ^18 || ^19", "react-dom": "^17 || ^18 || ^19" }, "optionalPeers": ["@types/react"] }, "sha512-x/PDDCYzoqPpjrdyb3VcyylTI2IjUXEtYDGi5foh7KsnmNJIIaVwA2GLgDH1dps1GgXiJbA60hM+AyuTfQzIvw=="], @@ -256,6 +285,10 @@ "@fontsource/eb-garamond": ["@fontsource/eb-garamond@5.3.0", "", {}, "sha512-VplwHpB8FLuzl+4G5LDci1b3Z4SghiuXH8XEgaxjnOb5aue5mEt1njmDKeO6iyLeGa73LffU7PC7hzncLRx0ZA=="], + "@grpc/grpc-js": ["@grpc/grpc-js@1.14.4", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ=="], + + "@grpc/proto-loader": ["@grpc/proto-loader@0.7.15", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ=="], + "@hono/node-server": ["@hono/node-server@1.19.17", "", { "peerDependencies": { "hono": "^4" } }, "sha512-dSneS5qhiauZWGDCeK4o695Xd9nUNjviSZCMQrj10eetr8Uln1ucn6bbphOM6UynAMMtNIzZNSpL9vnASJwrPQ=="], "@joshwooding/vite-plugin-react-docgen-typescript": ["@joshwooding/vite-plugin-react-docgen-typescript@0.7.0", "", { "dependencies": { "glob": "^13.0.1", "react-docgen-typescript": "^2.2.2" }, "peerDependencies": { "typescript": ">= 4.3.x", "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["typescript"] }, "sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ=="], @@ -270,6 +303,8 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], + "@mdx-js/react": ["@mdx-js/react@3.1.1", "", { "dependencies": { "@types/mdx": "^2.0.0" }, "peerDependencies": { "@types/react": ">=16", "react": ">=16" } }, "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw=="], "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], @@ -286,6 +321,8 @@ "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="], + "@openrouter/ai-sdk-provider": ["@openrouter/ai-sdk-provider@3.0.0", "", { "peerDependencies": { "ai": "^7.0.0", "zod": "^3.25.76 || ^4.1.8" } }, "sha512-m9XTSWoODH2RM5OsZpaGiN7QRR8cdP5paBWq699Tu3JVmGPBKT8xF8XwV0ZBVVsjikD/JgWfak4VSsTR4wAVbg=="], + "@oxc-project/runtime": ["@oxc-project/runtime@0.146.0", "", {}, "sha512-lbXHIpZ1MmK6zuw5txlMdIZ2waLVUIU5Gnm3sEuwJOiqDfQfbtjeHscatmeBoxbv8+If9LFM6PGh/3DcDWYIYw=="], "@oxc-project/types": ["@oxc-project/types@0.146.0", "", {}, "sha512-XC0QsnnhVe7sLIWmYmdPw7x5P0h4W8vUU3Nv1ySgWXtvCz8NizoAEpGXA0sOYoJQV2Rl13LgURAHQ5cI5ILCSA=="], @@ -382,6 +419,24 @@ "@polka/url": ["@polka/url@1.0.0-next.29", "", {}, "sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww=="], + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.5", "", {}, "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.1", "", {}, "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.1", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1" } }, "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.2", "", {}, "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug=="], + "@radix-ui/colors": ["@radix-ui/colors@3.0.0", "", {}, "sha512-FUOsGBkHrYJwCSEtWRCIfQbZG7q1e6DgxCIOe1SUQzDe/7rXXeA47s8yCn6fuTNQAj1Zq4oTFi9Yjp3wzElcxg=="], "@radix-ui/number": ["@radix-ui/number@1.1.0", "", {}, "sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ=="], @@ -544,6 +599,10 @@ "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + "@types/docker-modem": ["@types/docker-modem@3.0.6", "", { "dependencies": { "@types/node": "*", "@types/ssh2": "*" } }, "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg=="], + + "@types/dockerode": ["@types/dockerode@4.0.1", "", { "dependencies": { "@types/docker-modem": "*", "@types/node": "*", "@types/ssh2": "*" } }, "sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q=="], + "@types/doctrine": ["@types/doctrine@0.0.9", "", {}, "sha512-eOIHzCUSH7SMfonMG1LsC2f8vxBFtho6NGBznK41R84YzPuvSBzrhEps33IsQiOW9+VL6NQ9DbjQJznk/S4uRA=="], "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], @@ -558,6 +617,8 @@ "@types/resolve": ["@types/resolve@1.20.6", "", {}, "sha512-A4STmOXPhMUtHH+S6ymgE2GiBSMqf4oTvcQZMcHzokuTLVYzXTB8ttjcgxOVaAp2lGwEdzZ0J+cRbbeevQj1UQ=="], + "@types/ssh2": ["@types/ssh2@1.15.6", "", { "dependencies": { "@types/node": "^18.11.18" } }, "sha512-oGdxhBqcRTwSTKFm+9EiKzkNVYRLEFkcW44lhguvBalGJbWfGnDt/ezwSUZc+SF9m9bMc3VyklNAtp7zICjS5w=="], + "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], @@ -600,6 +661,8 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "@vercel/oidc": ["@vercel/oidc@3.2.0", "", {}, "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug=="], + "@vitejs/plugin-react": ["@vitejs/plugin-react@6.1.1", "", { "dependencies": { "@rolldown/pluginutils": "^1.0.1" }, "peerDependencies": { "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", "babel-plugin-react-compiler": "^1.0.0", "oxc-transform-react": "^0.145.0", "vite": "^8.0.0" }, "optionalPeers": ["@rolldown/plugin-babel", "babel-plugin-react-compiler", "oxc-transform-react"] }, "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw=="], "@vitest/browser": ["@vitest/browser@4.1.11", "", { "dependencies": { "@blazediff/core": "1.9.1", "@vitest/mocker": "4.1.11", "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pngjs": "^7.0.0", "sirv": "^3.0.2", "tinyrainbow": "^3.1.0", "ws": "^8.19.0" }, "peerDependencies": { "vitest": "4.1.11" } }, "sha512-bwMovvAeuTFOK5kIFevw4VEf+1gVEICv4SYK4k3knJOxl6b1zEWud8mYKD73e1B0odAn174h1MofURy2TPWf3w=="], @@ -640,6 +703,8 @@ "@webcontainer/env": ["@webcontainer/env@1.1.1", "", {}, "sha512-6aN99yL695Hi9SuIk1oC88l9o0gmxL1nGWWQ/kNy81HigJ0FoaoTXpytCj6ItzgyCEwA9kF1wixsTuv5cjsgng=="], + "@workflow/serde": ["@workflow/serde@4.1.0", "", {}, "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ=="], + "@yuku-codegen/binding-darwin-arm64": ["@yuku-codegen/binding-darwin-arm64@0.5.48", "", { "os": "darwin", "cpu": "arm64" }, "sha512-yo96Oef12WzqnphInfz/eexVse3+kWgfGS5g2S3rFS3dcGn1ENW9xLFDZUP9rh+yP76DOq38wBoFi1+I9+6qBg=="], "@yuku-codegen/binding-darwin-x64": ["@yuku-codegen/binding-darwin-x64@0.5.48", "", { "os": "darwin", "cpu": "x64" }, "sha512-aRCTw0EZC4bVosmw//0OMYP5tGWFE0Cu5yUBFkUbhXx/iBzvORcJ2xPNlOp/vtCCo9Ys4vp8b0DigJV6uOVb2g=="], @@ -690,6 +755,8 @@ "acorn": ["acorn@8.18.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ=="], + "ai": ["ai@7.0.93", "", { "dependencies": { "@ai-sdk/gateway": "4.0.75", "@ai-sdk/provider": "4.0.10", "@ai-sdk/provider-utils": "5.0.36" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CJss6zb9mlltk/mCr8qom20NBnqEQxVawkqwtT62tCwsxilZpXfHNRMRwcS3XRpzdP1kEluVuDBUayYWEEd95g=="], + "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], @@ -704,6 +771,8 @@ "askgina-evals-web": ["askgina-evals-web@workspace:apps/evals"], + "asn1": ["asn1@0.2.6", "", { "dependencies": { "safer-buffer": "~2.1.0" } }, "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], "ast-types": ["ast-types@0.16.3", "", { "dependencies": { "tslib": "^2.0.1" } }, "sha512-FvWoWYfSCM6kRxCSH+MGLHIKKGRL6A6AW7Zek2O32REPQRdg131428uRTKMBYAeRd3XXAaHDS60Wpri7CdKDrA=="], @@ -712,14 +781,24 @@ "balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + "baseline-browser-mapping": ["baseline-browser-mapping@2.11.21", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ=="], + "bcrypt-pbkdf": ["bcrypt-pbkdf@1.0.2", "", { "dependencies": { "tweetnacl": "^0.14.3" } }, "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w=="], + + "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], + "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], "brace-expansion": ["brace-expansion@5.0.9", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg=="], "browserslist": ["browserslist@4.28.9", "", { "dependencies": { "baseline-browser-mapping": "^2.11.20", "caniuse-lite": "^1.0.30001810", "electron-to-chromium": "^1.5.420", "node-releases": "^2.0.54", "update-browserslist-db": "^1.3.2" }, "bin": { "browserslist": "cli.js" } }, "sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg=="], + "buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], + + "buildcheck": ["buildcheck@0.0.7", "", {}, "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA=="], + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], "bundle-name": ["bundle-name@4.1.0", "", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], @@ -736,10 +815,18 @@ "check-error": ["check-error@2.1.3", "", {}, "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA=="], + "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + "class-variance-authority": ["class-variance-authority@0.7.1", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="], + "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], + "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="], + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + "content-disposition": ["content-disposition@1.1.0", "", {}, "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g=="], "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], @@ -752,6 +839,8 @@ "cors": ["cors@2.8.6", "", { "dependencies": { "object-assign": "^4", "vary": "^1" } }, "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw=="], + "cpu-features": ["cpu-features@0.0.10", "", { "dependencies": { "buildcheck": "~0.0.6", "nan": "^2.19.0" } }, "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], "css.escape": ["css.escape@1.5.1", "", {}, "sha512-YUifsXXuknHlUsmlgyY0PKzgPOr7/FjCePfHNt0jxm83wHZi44VDMQ7/fGNkjY3/jV1MC+1CmZbaHzugyeRtpg=="], @@ -776,6 +865,10 @@ "detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="], + "docker-modem": ["docker-modem@5.0.7", "", { "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", "split-ca": "^1.0.1", "ssh2": "^1.15.0" } }, "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA=="], + + "dockerode": ["dockerode@4.0.12", "", { "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", "tar-fs": "^2.1.4", "uuid": "^10.0.0" } }, "sha512-/bCZd6KlGcjZO8Buqmi/vXuqEGVEZ0PNjx/biBNqJD3MhK9DmdiAuKxqfNhflgDESDIiBz3qF+0e55+CpnrUcw=="], + "doctrine": ["doctrine@3.0.0", "", { "dependencies": { "esutils": "^2.0.2" } }, "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w=="], "dom-accessibility-api": ["dom-accessibility-api@0.6.3", "", {}, "sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w=="], @@ -788,10 +881,14 @@ "electron-to-chromium": ["electron-to-chromium@1.5.422", "", {}, "sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA=="], + "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + "empathic": ["empathic@2.0.1", "", {}, "sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q=="], "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "enhanced-resolve": ["enhanced-resolve@5.24.5", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A=="], "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], @@ -840,12 +937,16 @@ "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], + "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], "get-nonce": ["get-nonce@1.0.1", "", {}, "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q=="], @@ -862,12 +963,14 @@ "hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], - "hono": ["hono@4.13.4", "", {}, "sha512-AGEwKIyRMHRv1t8Wjwa3LHxQ61X5CqrdFT+4BRNTpqS5aJNnpl5WLjADb7vFlJzI/8uK7T5QLVApCMQKNa3LgQ=="], + "hono": ["hono@4.13.7", "", {}, "sha512-c8/gF9ac8Y78/agExVocyLevgR+JlpNB444Py0FSX8pJoPdYUfUzRcXtYEYGwt6l19qIlVZPN5Mfsw9jFShmQQ=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], "iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + "indent-string": ["indent-string@4.0.0", "", {}, "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], @@ -880,6 +983,8 @@ "is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], @@ -896,6 +1001,8 @@ "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], + "json-schema": ["json-schema@0.4.0", "", {}, "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA=="], + "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], "json-schema-typed": ["json-schema-typed@8.0.2", "", {}, "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA=="], @@ -926,6 +1033,10 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.33.0", "", { "os": "win32", "cpu": "x64" }, "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA=="], + "lodash.camelcase": ["lodash.camelcase@4.3.0", "", {}, "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="], + + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "loupe": ["loupe@3.2.1", "", {}, "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ=="], "lru-cache": ["lru-cache@11.5.2", "", {}, "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g=="], @@ -954,6 +1065,8 @@ "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], + "mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="], + "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -962,6 +1075,8 @@ "msgpackr-extract": ["msgpackr-extract@3.0.4", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw=="], + "nan": ["nan@2.28.0", "", {}, "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ=="], + "nanoid": ["nanoid@3.3.18", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w=="], "negotiator": ["negotiator@1.1.0", "", { "dependencies": { "content-type": "^2.1.0" } }, "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg=="], @@ -1014,8 +1129,12 @@ "pretty-format": ["pretty-format@27.5.1", "", { "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", "react-is": "^17.0.1" } }, "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ=="], + "protobufjs": ["protobufjs@7.6.6", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg=="], + "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], + "pure-rand": ["pure-rand@8.4.2", "", {}, "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng=="], "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], @@ -1040,10 +1159,14 @@ "react-style-singleton": ["react-style-singleton@2.2.3", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "recast": ["recast@0.23.21", "", { "dependencies": { "ast-types": "^0.16.1", "esprima": "~4.0.0", "source-map": "~0.6.1", "tiny-invariant": "^1.3.3", "tslib": "^2.0.1" } }, "sha512-mFAyJq9vUbSTARLZUvAEf1z3YxlvAwswbmxMx2mPA/MSm4KmpwvwvhsH/NIrZhyOuwD60Lzyw2qh83uCbgTPYw=="], "redent": ["redent@3.0.0", "", { "dependencies": { "indent-string": "^4.0.0", "strip-indent": "^3.0.0" } }, "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg=="], + "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], "reselect": ["reselect@5.3.0", "", {}, "sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg=="], @@ -1056,6 +1179,8 @@ "run-applescript": ["run-applescript@7.1.0", "", {}, "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q=="], + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], @@ -1088,6 +1213,10 @@ "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "split-ca": ["split-ca@1.0.1", "", {}, "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ=="], + + "ssh2": ["ssh2@1.17.0", "", { "dependencies": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2" }, "optionalDependencies": { "cpu-features": "~0.0.10", "nan": "^2.23.0" } }, "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ=="], + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], @@ -1096,6 +1225,12 @@ "storybook": ["storybook@10.3.6", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.1", "@testing-library/jest-dom": "^6.9.1", "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", "@webcontainer/env": "^1.1.1", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0", "open": "^10.2.0", "recast": "^0.23.5", "semver": "^7.7.3", "use-sync-external-store": "^1.5.0", "ws": "^8.18.0" }, "peerDependencies": { "prettier": "^2 || ^3", "vite-plus": "^0.1.15" }, "optionalPeers": ["prettier", "vite-plus"], "bin": "./dist/bin/dispatcher.js" }, "sha512-vbSz7g/1rGMC1uAULqMZjALkIuLu2QABqfhRYhyr/11kzyesi+vAmwyJLukZP1FfecxGOgMwOh6GS0YsGpHAvQ=="], + "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + + "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], "strip-indent": ["strip-indent@4.1.1", "", {}, "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA=="], @@ -1110,6 +1245,10 @@ "tapable": ["tapable@2.3.3", "", {}, "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A=="], + "tar-fs": ["tar-fs@2.1.5", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw=="], + + "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + "tiny-invariant": ["tiny-invariant@1.3.3", "", {}, "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg=="], "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], @@ -1134,12 +1273,16 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "tweetnacl": ["tweetnacl@0.14.5", "", {}, "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="], + "type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="], "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "typescript-api": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "undici": ["undici@7.29.1", "", {}, "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q=="], + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], @@ -1154,6 +1297,10 @@ "use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + + "uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], "vite": ["vite@8.2.2", "", { "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.5", "postcss": "^8.5.26", "rolldown": "~1.2.4", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.4.0 || ^0.5.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q=="], @@ -1168,30 +1315,44 @@ "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "ws": ["ws@8.21.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw=="], "wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], + "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], "yaml": ["yaml@2.8.3", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg=="], + "yargs": ["yargs@17.7.3", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g=="], + + "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], + "yuku-codegen": ["yuku-codegen@0.5.48", "", { "dependencies": { "@yuku-toolchain/types": "0.5.43" }, "optionalDependencies": { "@yuku-codegen/binding-darwin-arm64": "0.5.48", "@yuku-codegen/binding-darwin-x64": "0.5.48", "@yuku-codegen/binding-freebsd-x64": "0.5.48", "@yuku-codegen/binding-linux-arm-gnu": "0.5.48", "@yuku-codegen/binding-linux-arm-musl": "0.5.48", "@yuku-codegen/binding-linux-arm64-gnu": "0.5.48", "@yuku-codegen/binding-linux-arm64-musl": "0.5.48", "@yuku-codegen/binding-linux-x64-gnu": "0.5.48", "@yuku-codegen/binding-linux-x64-musl": "0.5.48", "@yuku-codegen/binding-win32-arm64": "0.5.48", "@yuku-codegen/binding-win32-x64": "0.5.48" } }, "sha512-p7HxD5Xl4jzDzqMrGePAOeSHmRY4g58h4HuGq15weQFPxuPWd/W6e7nqp/+Lea6JfpOdBwJOAyXFqIZ/J9Zfnw=="], "yuku-parser": ["yuku-parser@0.5.48", "", { "dependencies": { "@yuku-toolchain/types": "0.5.43" }, "optionalDependencies": { "@yuku-parser/binding-darwin-arm64": "0.5.48", "@yuku-parser/binding-darwin-x64": "0.5.48", "@yuku-parser/binding-freebsd-x64": "0.5.48", "@yuku-parser/binding-linux-arm-gnu": "0.5.48", "@yuku-parser/binding-linux-arm-musl": "0.5.48", "@yuku-parser/binding-linux-arm64-gnu": "0.5.48", "@yuku-parser/binding-linux-arm64-musl": "0.5.48", "@yuku-parser/binding-linux-x64-gnu": "0.5.48", "@yuku-parser/binding-linux-x64-musl": "0.5.48", "@yuku-parser/binding-win32-arm64": "0.5.48", "@yuku-parser/binding-win32-x64": "0.5.48" } }, "sha512-OWBfhrpgK9+/4+IXG9oT8Bao4AhViQA7vdyNNH7EUg8dQYgwa70XtIBWTpCEme1P1ECyoDNYkn0wT63f8XRcVA=="], - "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], + "@askgina/plugin-core/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "@babel/core/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "@babel/helper-compilation-targets/lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], "@babel/helper-compilation-targets/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], + "@grpc/grpc-js/@grpc/proto-loader": ["@grpc/proto-loader@0.8.1", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg=="], + + "@modelcontextprotocol/sdk/zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "@radix-ui/react-collection/@radix-ui/react-context": ["@radix-ui/react-context@1.1.0", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A=="], "@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.1.0", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-FUCf5XMfmW4dtYl69pdS4DbxKy8nj4M7SafBgPllysxmdachynNflAdp/gCsnYWNDnge6tI9onzMp5ARYc1KNw=="], @@ -1222,6 +1383,8 @@ "@testing-library/dom/dom-accessibility-api": ["dom-accessibility-api@0.5.16", "", {}, "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg=="], + "@types/ssh2/@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], + "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], "negotiator/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], @@ -1234,6 +1397,8 @@ "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], + "wrap-ansi/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "@tailwindcss/node/lightningcss/lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], "@tailwindcss/node/lightningcss/lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], @@ -1256,6 +1421,8 @@ "@tailwindcss/node/lightningcss/lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="], + "@types/ssh2/@types/node/undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "storybook/@vitest/expect/@vitest/utils": ["@vitest/utils@3.2.4", "", { "dependencies": { "@vitest/pretty-format": "3.2.4", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" } }, "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA=="], "storybook/@vitest/expect/chai": ["chai@5.3.3", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw=="], diff --git a/package.json b/package.json index a6be495..579484e 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,9 @@ "eval:export-public": "bun run build && bun packages/evals/dist/bin/export-public-results.js", "eval:responses": "bun run build && bun packages/evals/dist/bin/live.js --runner responses", "eval:codex": "bun run build && bun packages/evals/dist/bin/live.js --runner codex", + "eval:openrouter": "bun run build && bun packages/evals/dist/bin/live.js --runner openrouter", + "eval:claude": "bun run build && bun packages/evals/dist/bin/live.js --runner claude", + "eval:omp": "bun run build && bun packages/evals/dist/bin/live.js --runner omp", "check:marketplace:codex": "bun run build && bun packages/evals/dist/bin/check-codex-marketplace.js", "effect-tsgo:patch": "bun scripts/effect-tsgo-patch-if-needed.ts", "prepare": "bun run effect-tsgo:patch", @@ -57,6 +60,7 @@ }, "overrides": { "@effect/platform-node-shared": "4.0.0-rc.111", + "dockerode>uuid": "11.1.1", "esbuild": "0.28.1", "oxlint": "1.78.0", "qs": "6.16.0" diff --git a/packages/contracts/__tests__/eval-results.test.ts b/packages/contracts/__tests__/eval-results.test.ts index 4fe4e57..2db593d 100644 --- a/packages/contracts/__tests__/eval-results.test.ts +++ b/packages/contracts/__tests__/eval-results.test.ts @@ -1,12 +1,15 @@ import { createHash } from "node:crypto"; import { assert, describe, it } from "@effect/vitest"; -import { Effect } from "effect"; +import { Effect, Schema } from "effect"; import { type PublicEvalAttemptCapture, type PublicEvalAttemptSummary, type PublicEvalIndex, type PublicEvalPublication, type PublicEvalResult, + PUBLIC_EVAL_DECODE_OPTIONS, + PublicEvalIdentifierSchema, + PublicEvalModelSchema, decodePublicEvalAttemptCapture, decodePublicEvalIndex, decodePublicEvalPublication, @@ -387,4 +390,52 @@ describe("@askgina/contracts public eval boundaries", () => { ); }), ); + + it.effect("keeps slash-separated model identity while identifiers reject path-like values", () => + Effect.gen(function* () { + const decodeIdentifier = Schema.decodeUnknownEffect( + PublicEvalIdentifierSchema, + PUBLIC_EVAL_DECODE_OPTIONS, + ); + const decodeModel = Schema.decodeUnknownEffect( + PublicEvalModelSchema, + PUBLIC_EVAL_DECODE_OPTIONS, + ); + const accepted = ["openai/gpt-5.1", "openrouter/openai/gpt-5.1", "synthetic-model"] as const; + for (const model of accepted) { + assert.strictEqual(yield* decodeModel(model), model); + const decoded = yield* decodePublicEvalResult({ + ...RESULT, + configuration: { ...RESULT.configuration, model }, + }); + assert.strictEqual(decoded.configuration.model, model); + } + + yield* rejected(decodeIdentifier("openai/gpt-5.1")); + yield* rejected( + decodePublicEvalResult({ + ...RESULT, + resultId: "openai/gpt-5.1", + }), + ); + + const rejectedModels = [ + "/tmp", + "../foo", + "a//b", + "https://example.com/model", + "openai/gpt-5.1?q=1", + "a".repeat(129), + ] as const; + for (const model of rejectedModels) { + yield* rejected(decodeModel(model)); + yield* rejected( + decodePublicEvalResult({ + ...RESULT, + configuration: { ...RESULT.configuration, model }, + }), + ); + } + }), + ); }); diff --git a/packages/contracts/src/eval-results.ts b/packages/contracts/src/eval-results.ts index 03b18b5..267d76a 100644 --- a/packages/contracts/src/eval-results.ts +++ b/packages/contracts/src/eval-results.ts @@ -14,6 +14,8 @@ import { Function, Schema, type SchemaAST } from "effect"; */ const IDENTIFIER = /^[A-Za-z0-9](?:[A-Za-z0-9._:-]{0,126}[A-Za-z0-9])?$/u; +const MODEL_IDENTIFIER = + /^[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?(?:\/[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?)*$/u; const SHA_256 = /^[a-f0-9]{64}$/u; const ATTEMPT_ID = /^attempt-[a-f0-9]{64}$/u; const UTC_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{3})?Z$/u; @@ -47,6 +49,10 @@ export const PublicEvalIdentifierSchema = Schema.NonEmptyString.check( Schema.isMaxLength(128), Schema.isPattern(IDENTIFIER), ); +export const PublicEvalModelSchema = Schema.NonEmptyString.check( + Schema.isMaxLength(128), + Schema.isPattern(MODEL_IDENTIFIER), +); export const PublicEvalSha256Schema = Schema.String.check(Schema.isPattern(SHA_256)); export const PublicEvalTimestampSchema = Schema.NonEmptyString.check( Schema.isMaxLength(64), @@ -347,7 +353,7 @@ export const PublicEvalResultSchema = Schema.Struct({ configuration: Schema.Struct({ availability: Schema.Literals(["pinned", "labels_only"]), candidate: PublicEvalIdentifierSchema, - model: PublicEvalIdentifierSchema, + model: PublicEvalModelSchema, reasoning: Schema.NullOr(PublicEvalIdentifierSchema), pinnedSha256: Schema.NullOr(PublicEvalSha256Schema), }), @@ -761,7 +767,7 @@ const PublicEvalIndexEntrySchema = Schema.Struct({ Schema.Struct({ suiteId: PublicEvalIdentifierSchema, candidate: PublicEvalIdentifierSchema, - model: PublicEvalIdentifierSchema, + model: PublicEvalModelSchema, startedAt: PublicEvalTimestampSchema, }), ), diff --git a/packages/evals/README.md b/packages/evals/README.md index 0fe77b4..64675b3 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -1,11 +1,16 @@ # @askgina/evals -One schema and rubric drive hermetic replay, OpenAI Responses API trials, and -Codex CLI trials. Live runners use the same suite cases, model, reasoning mode, -case selection, repetition count, and sanitized aggregate shape. +One schema and rubric drive hermetic replay, OpenAI Responses API trials, +OpenRouter trials, Codex CLI trials, Claude CLI trials, and OMP HarnessAgent +trials. Live runners use the same suite cases, reasoning mode, case selection, +repetition count, and sanitized aggregate shape. Model IDs and turn limits remain +backend-specific. `@askgina/evals` is a Bun 1.4.x-only compiled `dist` package. The root -`eval:replay`, `eval:responses`, and `eval:codex` commands build the package graph, then execute `packages/evals/dist/bin/*.js`; suite and observation YAML remain repository inputs. +`eval:replay`, `eval:responses`, `eval:codex`, `eval:openrouter`, +`eval:claude`, and `eval:omp` commands build the package graph, then execute +`packages/evals/dist/bin/*.js`; suite and observation YAML remain repository +inputs. Artifact verification clean-installs the built tarball and exercises its compiled import and replay entrypoint. The package supports only its root ESM import; Node.js, CommonJS, browser and edge runtimes, and subpath imports are unsupported. @@ -64,29 +69,49 @@ and index JSON at `/#/handoff`, using erased contract types, not evaluator runti ## Live trials -Live commands require a clean Git worktree, three to five repetitions, and the -same `--suite`, `--model`, `--reasoning`, and `--timeout-ms` values when comparing -runners. The default live benchmark suite is `ask-gina-routing-smoke.yaml`. Both -runners require `ASK_GINA_ACCESS_TOKEN` and `OPENAI_API_KEY` in the process -environment. Codex trials additionally require an absolute executable path in -`CODEX_EVAL_EXECUTABLE` and its lowercase SHA-256 digest in -`CODEX_EVAL_EXECUTABLE_SHA256`. On Linux, the runner rejects group- or -world-writable inputs, copies the verified bytes into a private non-writable -snapshot, unlinks it, and launches its open descriptor; unsupported platforms fail -closed. It then installs and validates the repository plugin under a fresh temporary -`CODEX_HOME`, seeds only -the temporary Gina MCP credential, verifies the exact MCP endpoint and OAuth -status and production catalog, then runs with an enforced permission profile. -The profile denies reads from `CODEX_HOME`, read-allows only the minimal runtime, -empty trial working tree, and validated plugin skills, disables shell network and -web search, and enables only the observed Gina MCP tools. Trials also use no -approvals, ignored user/project rules, bounded output, and a minimal child -environment. +Live commands require a clean Git worktree and three to five repetitions. Use the +same suite, case selection, reasoning setting, repetition count, account class, +and timeout for a controlled comparison. Model IDs use each backend's namespace, +so the literal `--model` value may differ for the same model family. Record that +mapping with the result instead of presenting the strings as identical settings. +The default live benchmark suite is `ask-gina-routing-smoke.yaml`. + +Every live runner requires `ASK_GINA_ACCESS_TOKEN`. The provider and native CLI +credentials differ: + +| Runner | Additional environment | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Responses API | `OPENAI_API_KEY` | +| OpenRouter | `OPENROUTER_API_KEY` | +| Codex CLI | `OPENAI_API_KEY`, absolute `CODEX_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `CODEX_EVAL_EXECUTABLE_SHA256` | +| Claude CLI | `ANTHROPIC_API_KEY` and absolute `CLAUDE_EVAL_EXECUTABLE` | +| OMP harness | `OMP_EVAL_API_KEY`, absolute `OMP_EVAL_EXECUTABLE`, its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256`, and a local Docker engine | + +The supported native Codex, Claude, and OMP paths use explicit API keys in +isolated evaluation homes. They do not reuse a saved personal login. Saved-login +reuse, refresh ownership, and personal-home integration remain deferred. OMP does +not read `~/.omp` or inherit `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / +`OPENROUTER_API_KEY`. Native OMP also never exports those names into the +isolated child. The child receives only `OMP_EVAL_PROVIDER_API_KEY` for +the private `omp-eval` provider. + +Responses and Codex accept model IDs understood by their OpenAI backends, such as +`gpt-5.1`. OpenRouter uses its `provider/model` namespace, such as +`openai/gpt-5.1`. Claude uses a Claude CLI model ID or alias without the +`anthropic/` prefix, such as `claude-sonnet-4-5-20250929`. OMP requires `--provider` +to select `openai`, `anthropic`, or `openrouter`; `--model` uses that provider's +backend ID. For example, `--provider openai --model gpt-5.1` records +`openai/gpt-5.1`, while `--provider openrouter --model openai/gpt-5.1` records +`openrouter/openai/gpt-5.1`. The report, attempt input, and observation keep that +same identity. There is no separate displayed model. These examples show the +required ID shapes. +They do not declare a benchmark configuration or claim that a live run was +performed. ```sh bun run eval:responses -- \ --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ - --run-id 2026-08-25-main \ + --run-id local-responses-example \ --candidate main \ --model gpt-5.1 \ --reasoning medium \ @@ -96,8 +121,41 @@ bun run eval:responses -- \ bun run eval:codex -- \ --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ - --run-id 2026-08-25-main \ + --run-id local-codex-example \ + --candidate main \ + --model gpt-5.1 \ + --reasoning medium \ + --repetitions 3 \ + --account-class eval \ + --timeout-ms 120000 + +bun run eval:openrouter -- \ + --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ + --run-id local-openrouter-example \ + --candidate main \ + --model openai/gpt-5.1 \ + --reasoning medium \ + --repetitions 3 \ + --account-class eval \ + --timeout-ms 120000 \ + --max-steps 8 + +bun run eval:claude -- \ + --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ + --run-id local-claude-example \ + --candidate main \ + --model claude-sonnet-4-5-20250929 \ + --reasoning medium \ + --repetitions 3 \ + --account-class eval \ + --timeout-ms 120000 \ + --max-turns 8 + +bun run eval:omp -- \ + --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ + --run-id local-omp-example \ --candidate main \ + --provider openai \ --model gpt-5.1 \ --reasoning medium \ --repetitions 3 \ @@ -105,11 +163,104 @@ bun run eval:codex -- \ --timeout-ms 120000 ``` -Repeat `--case ` to run a strict subset. `--timeout-ms` is required so -both runners share the same per-trial budget. Secrets have no command-line flags. +Repeat `--case ` to run a strict subset. `--timeout-ms` is required for +the per-trial budget. `--max-steps` is optional only for OpenRouter, and +`--max-turns` is optional only for Claude. Both default to `8` and accept `1` to +`32`. `--provider` is required only for OMP. The other runners reject those +flags. OMP rejects `--max-steps` and `--max-turns`. ACP does not expose a +portable native model-step boundary, so OMP does not claim an equal step budget +with OpenRouter or Claude. Secrets have no command-line flags. +A missing required flag or backend credential fails closed. The CLI does not +switch runners or auth methods. Add +`--attempts-output /tmp/eval-private/attempts.json` to any command when retained, +report-bound attempt summaries are required. The capture rules in the previous +section still apply. + +OpenRouter executes Gina tools through the local AI SDK MCP client. Responses +uses OpenAI-hosted MCP. Neither proves native plugin activation, and these two +execution paths retain separate runner identities. Codex and Claude load the +repository plugin through their native CLIs. OMP talks to `omp acp` through +HarnessAgent in Docker and keeps the `omp_harness` target distinct. Task +conformance and observed plugin activation are scored separately. The Claude +adapter has offline verification only. Synthetic events, loopback model +fixtures, and OMP Docker/runtime proof do not establish measured native-plugin +activation; Claude's live activation remains unverified. + +Claude requires a CLI supporting `--restricted` and `--permission-prompts none`; +the flag set was checked against version `2.1.263`. Bare mode is not used because +it suppresses the native Skill tool in this version. +The runner stages a complete session-only plugin and keeps its credentials in a +separate temporary config directory. It never installs into your personal home. +The isolated child sets `CLAUDE_CODE_MAX_RETRIES=0` and +`CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK=1`, and does not inherit the retry +watchdog. These disable the main API retry budget and non-streaming fallback. +Claude documents additional pre-response stall/drop reissues outside that budget, +so this is not a guarantee of one HTTP request per model step. The trial deadline +still bounds the native process. +`--max-steps` bounds OpenRouter generation steps, and `--max-turns` bounds Claude +turns. Neither is a count of individual tool calls. + +For Codex, the runner verifies and snapshots the configured executable on Linux, +installs and validates the repository plugin under a fresh temporary `CODEX_HOME`, +and seeds only the temporary Gina MCP credential. It verifies the production MCP +endpoint, OAuth status, and catalog before the trial. The enforced profile denies +reads from `CODEX_HOME`, allows only the minimal runtime, empty trial working tree, +and validated plugin skills, disables shell network and web search, and enables +only the observed Gina MCP tools. Trials also use no approvals, ignored +user/project rules, bounded output, and a minimal child environment. Unsupported +platforms fail closed. + +OMP requires the tested `18.1.14` executable and a local Docker engine at +`/var/run/docker.sock`. The CLI verifies its SHA-256 and snapshots it once, then +starts a fresh Docker session per trial. The default Node image is digest-pinned. +Containers run as a non-root user, with a read-only root filesystem and runtime +mount, writable temporary filesystems, and no host Docker socket or personal OMP +configuration mounted inside. Bootstrap installs the pinned ACP bridge dependencies. + +Each trial writes a static `models.yml` with one private `omp-eval` provider +and one selected-model entry. Public `--provider` / `--model` identity stays +on the report, attempt, and observation. Native ACP uses `--provider omp-eval` +and the original backend model id, including OpenRouter nested slugs. The +child credential env is only `OMP_EVAL_PROVIDER_API_KEY`; the credential +transformation hook matches that same name. Standard provider env names are +not exported, so built-in discovery managers do not start. + +OpenAI and OpenRouter use Chat Completions, not the Responses transport that +OMP can select for built-in providers. Anthropic uses its Messages API. The +CLI's `--reasoning` value selects native `--thinking`, with explicit effort +settings for OpenAI/OpenRouter and thinking budgets for Anthropic. The model +entry leaves capacities, cost, and input modalities to OMP 18.1.14's bundled +same-id metadata or defaults for unknown model ids. + +This local Docker provider does not broker credentials outside the container. +The Gina bearer stays on the host, where canonical MCP reads execute. The native +guard permits only the canonical MCP inventory and exact staged skill reads; +URL reads, other files, shell tools, and unregistered tool attempts fail the trial. +Native intent tracing is disabled. ACP mappings use OMP's wire names, with a +`skill://` classifier for reads; the guard still checks the exact allowed URI. +The ACP launcher requires loaded guard evidence before forwarding the first prompt. +The guard waits up to five seconds for native MCP registration, within the trial +deadline, then requires the exact tool inventory before any model request. This +startup wait does not retry model or MCP calls. Native `retry.enabled` and +`retry.modelFallback` are false, disabling agent-level TurnRecovery retries and +configured model fallback. OMP 18.1.14's provider clients still retry some HTTP +errors independently of those settings. Native stream/stop recovery can also +reissue requests. There is no one-request guarantee; the absolute trial deadline +still applies. Host-side JSON Schema validation rejects invalid +arguments as forwarded by OMP before MCP execution. OMP may coerce the model's +raw arguments before this check. ACP does not provide a portable model-step limit. + +Skill activation requires a successful native read, not loaded metadata or an ACP +intent title. Token usage comes only from native guard evidence and remains absent +when unavailable. A successful result also requires completed native generation +and removal of the owned Docker resources. Failed or cancelled trials allow up to +eight seconds for cleanup without replacing the original failure. Synthetic model/MCP fixtures exercise +the real OMP process and protocol; they do not prove real model behavior, production +Gina connectivity, or measured native-plugin activation. Each live run writes exactly one mode-`0600` aggregate below the ignored `.plugin-eval-runs/` directory. Raw prompts, final answers, tool arguments, provider payloads, HTTP bodies, child output, and credential material are never persisted. A nonzero exit means the run failed or at least one rubric case did -not pass. +not pass. Exporting a measured result still requires the recorded manual approval +described above. Running or capturing attempts does not approve publication. diff --git a/packages/evals/__tests__/claude-cli.test.ts b/packages/evals/__tests__/claude-cli.test.ts new file mode 100644 index 0000000..148767a --- /dev/null +++ b/packages/evals/__tests__/claude-cli.test.ts @@ -0,0 +1,1202 @@ +import * as BunServices from "@effect/platform-bun/BunServices"; +import { listCatalogToolNames } from "@askgina/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import { Effect, Redacted, Schema } from "effect"; +import { TestClock } from "effect/testing"; + +import { + buildClaudeCliEnvironment, + parseClaudeCliStreamJson, + PluginEvalClaudeCliProcessError, + PluginEvalClaudeCliSpawnError, + PluginEvalClaudeCliTimeoutError, + runClaudeCliPluginEvalTrial, + type ClaudeCliCommand, + type ClaudeCliParseContext, + type ClaudeCliTrialRunner, +} from "../src/claude-cli"; +import type { PluginEvalCase } from "../src/contracts"; + +const PLUGIN_DIRECTORY = ["/tmp", "ask-gina-eval", "plugin"].join("/"); +const PLUGIN_SKILLS_DIRECTORY = `${PLUGIN_DIRECTORY}/skills`; +const WORKING_DIRECTORY = ["/tmp", "ask-gina-eval", "work"].join("/"); +const CONFIG_DIRECTORY = ["/tmp", "ask-gina-eval", "config"].join("/"); +const SKILL_PATH = `${PLUGIN_SKILLS_DIRECTORY}/research-spot-tokens/SKILL.md`; +const HOST_SKILL_PATH = ["/home", "eval", ".claude", "skills", "find-skills", "SKILL.md"].join("/"); +const TEST_API_KEY = Redacted.make("synthetic-anthropic-key"); +const TEST_MCP_AUTHORIZATION = Redacted.make("gina-read-secret"); + +const parseContext: ClaudeCliParseContext = { + pluginDirectory: PLUGIN_DIRECTORY, + workingDirectory: WORKING_DIRECTORY, + forbiddenReadRoots: [CONFIG_DIRECTORY], +}; + +const trialOptions = { + runId: "run-claude-read-only", + repetition: 1, + availableTools: listCatalogToolNames(), + workingDirectory: WORKING_DIRECTORY, + executablePath: "/opt/trusted/claude", + pluginDirectory: PLUGIN_DIRECTORY, + mcpAuthorization: TEST_MCP_AUTHORIZATION, + apiKey: TEST_API_KEY, + model: "claude-sonnet-4-5", + reasoning: "medium", +} as const; + +const evalCase: PluginEvalCase = { + id: "spot-direct-price", + category: "direct", + tags: ["spot"], + manual_priority: "required", + turns: [{ role: "user", content: "What is Ethereum trading at right now in USD?" }], + expected: { + skill: { kind: "exact", skill: "research-spot-tokens" }, + routing: { kind: "exact", tool: "spot.getSimplePrice" }, + }, +}; + +const fixtureResult = { + exitCode: 0, + stdout: "", + stdoutTruncated: false, + stderrTruncated: false, +} as const; + +const JsonLine = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)); + +const nativeMcpToolName = (canonical: string): string => + `mcp__ask-gina__${canonical.replace(/[^a-zA-Z0-9_-]/g, "_")}`; + +const NATIVE_PRICE_TOOL = nativeMcpToolName("spot.getSimplePrice"); +const NATIVE_ACCOUNT_TOOL = nativeMcpToolName("gina.getAccountAddresses"); + +const validInit = { + type: "system", + subtype: "init", + plugins: [{ name: "ask-gina", path: PLUGIN_DIRECTORY }], + mcp_servers: [{ name: "ask-gina", status: "connected" }], + tools: ["Skill", "Read", ...listCatalogToolNames().map(nativeMcpToolName)], +} as const; + +const collectPublicStrings = (value: unknown): readonly string[] => { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(collectPublicStrings); + if (value !== null && typeof value === "object") { + return Object.entries(value).flatMap(([key, nested]) => [key, ...collectPublicStrings(nested)]); + } + return []; +}; + +const jsonl = (...events: readonly Record[]): string => + `${events.map((event) => Schema.encodeUnknownSync(JsonLine)(event)).join("\n")}\n`; + +const successResult = { + type: "result", + subtype: "success", + is_error: false, + result: "ETH is $1.", + usage: { input_tokens: 10, output_tokens: 4, total_tokens: 14 }, +} as const; + +describe("Claude CLI JSONL evidence", () => { + it.effect("records only Skill events and plugin SKILL.md reads as activation", () => + Effect.sync(() => { + const parsed = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { type: "text", text: `Available skill: ${SKILL_PATH}` }, + { + type: "tool_use", + id: "skill_1", + name: "Skill", + input: { skill: "ask-gina:research-spot-tokens" }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "skill_1", content: "loaded" }], + }, + }, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "read_1", + name: "Read", + input: { file_path: SKILL_PATH }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "read_1", content: "# Skill" }], + }, + }, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "read_host", + name: "Read", + input: { file_path: HOST_SKILL_PATH }, + }, + ], + }, + }, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "mcp_1", + name: NATIVE_PRICE_TOOL, + input: { ids: "ethereum", vs_currencies: "usd" }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: "mcp_1", + content: { ethereum: { usd: 1 } }, + }, + ], + }, + }, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "mcp_plugin", + name: "mcp__plugin_ask-gina_gina__gina_getAccountAddresses", + input: {}, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "mcp_plugin", content: {} }], + }, + }, + successResult, + ), + parseContext, + ); + + assert.deepStrictEqual(parsed.activated_skills, ["research-spot-tokens"]); + assert.strictEqual(parsed.tool_calls.length, 1); + assert.strictEqual(parsed.tool_calls[0]?.name, "spot.getSimplePrice"); + assert.deepStrictEqual(parsed.tool_calls[0]?.arguments, { + ids: "ethereum", + vs_currencies: "usd", + }); + assert.strictEqual(parsed.unsupported_actions, 2); + assert.strictEqual(parsed.final_answer, "ETH is $1."); + assert.deepStrictEqual(parsed.token_usage, { + input_tokens: 10, + output_tokens: 4, + total_tokens: 14, + }); + assert.isFalse(parsed.malformed_jsonl); + assert.isFalse(parsed.incomplete); + }), + ); + + it.effect("omits unavailable or malformed usage without invalidating completion", () => + Effect.sync(() => { + const unavailable = parseClaudeCliStreamJson( + jsonl(validInit, { + type: "result", + subtype: "success", + is_error: false, + result: "ETH is $1.", + }), + parseContext, + ); + assert.isUndefined(unavailable.token_usage); + assert.isFalse(unavailable.malformed_jsonl); + assert.isFalse(unavailable.incomplete); + + const validCases = [ + { + usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + expected: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + }, + { + usage: { input_tokens: 10, output_tokens: 4 }, + expected: { input_tokens: 10, output_tokens: 4, total_tokens: 14 }, + }, + ]; + for (const { usage, expected } of validCases) { + const parsed = parseClaudeCliStreamJson( + jsonl(validInit, { ...successResult, usage }), + parseContext, + ); + assert.deepStrictEqual(parsed.token_usage, expected); + assert.isFalse(parsed.malformed_jsonl); + assert.isFalse(parsed.incomplete); + } + + const malformedCases = [ + { input_tokens: -1, output_tokens: 1, total_tokens: 0 }, + { input_tokens: 1, output_tokens: 0.5, total_tokens: 1 }, + { input_tokens: 1, output_tokens: 0, total_tokens: null }, + { input_tokens: 1, output_tokens: 0, total_tokens: Number.MAX_SAFE_INTEGER + 1 }, + { input_tokens: Number.MAX_SAFE_INTEGER, output_tokens: 1 }, + ]; + for (const usage of malformedCases) { + const parsed = parseClaudeCliStreamJson( + jsonl(validInit, { ...successResult, usage }), + parseContext, + ); + assert.isUndefined(parsed.token_usage); + assert.strictEqual(parsed.final_answer, "ETH is $1."); + assert.isFalse(parsed.malformed_jsonl); + assert.isFalse(parsed.incomplete); + } + }), + ); + + it.effect("does not treat injected skill-path text as activation evidence", () => + Effect.sync(() => { + const parsed = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "text", + text: `Use ${SKILL_PATH} and /research-spot-tokens before answering.`, + }, + { type: "thinking", thinking: "I should load research-spot-tokens" }, + ], + }, + }, + successResult, + ), + parseContext, + ); + + assert.deepStrictEqual(parsed.activated_skills, []); + assert.deepStrictEqual(parsed.tool_calls, []); + assert.strictEqual(parsed.unsupported_actions, 0); + assert.isUndefined(parsed.error); + }), + ); + + it.effect("records staged plugin SKILL.md reads only after a successful result", () => + Effect.sync(() => { + const pending = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { type: "tool_use", id: "read_1", name: "Read", input: { file_path: SKILL_PATH } }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(pending.activated_skills, []); + assert.isTrue(pending.incomplete); + + const failed = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { type: "tool_use", id: "read_1", name: "Read", input: { file_path: SKILL_PATH } }, + ], + }, + }, + { + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: "read_1", + is_error: true, + content: "missing", + }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(failed.activated_skills, []); + assert.isFalse(failed.incomplete); + + const activated = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { type: "tool_use", id: "read_1", name: "Read", input: { file_path: SKILL_PATH } }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "read_1", content: "# Skill" }], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(activated.activated_skills, ["research-spot-tokens"]); + assert.isFalse(activated.incomplete); + }), + ); + + it.effect("keeps MCP invocation order, rejects duplicate ids, and forbids config traversal", () => + Effect.sync(() => { + const reordered = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "mcp_a", + name: NATIVE_PRICE_TOOL, + input: { ids: "ethereum" }, + }, + { + type: "tool_use", + id: "mcp_b", + name: NATIVE_ACCOUNT_TOOL, + input: {}, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "mcp_b", content: {} }], + }, + }, + { + type: "user", + message: { + content: [ + { type: "tool_result", tool_use_id: "mcp_a", content: { ethereum: { usd: 1 } } }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.strictEqual(reordered.tool_calls[0]?.name, "spot.getSimplePrice"); + assert.strictEqual(reordered.tool_calls[0]?.sequence, 0); + assert.strictEqual(reordered.tool_calls[1]?.name, "gina.getAccountAddresses"); + assert.strictEqual(reordered.tool_calls[1]?.sequence, 1); + + const duplicate = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "dup", + name: NATIVE_PRICE_TOOL, + input: { ids: "ethereum" }, + }, + { + type: "tool_use", + id: "dup", + name: NATIVE_ACCOUNT_TOOL, + input: {}, + }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.isTrue(duplicate.malformed_jsonl); + + const traversal = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "read_config", + name: "Read", + input: { file_path: "../config/mcp.json" }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "read_config", content: "{}" }], + }, + }, + successResult, + ), + parseContext, + ); + assert.strictEqual(traversal.unsupported_actions, 1); + assert.deepStrictEqual(traversal.activated_skills, []); + }), + ); + + it.effect("scores Bash, WebFetch, and Write as unsupported actions", () => + Effect.sync(() => { + const parsed = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "bash_1", + name: "Bash", + input: { command: "curl example.com" }, + }, + { + type: "tool_use", + id: "fetch_1", + name: "WebFetch", + input: { url: "https://example.com" }, + }, + { + type: "tool_use", + id: "write_1", + name: "Write", + input: { file_path: "/tmp/out.txt" }, + }, + ], + }, + }, + successResult, + ), + parseContext, + ); + + assert.strictEqual(parsed.unsupported_actions, 3); + assert.include(parsed.error ?? "", "outside Skill"); + assert.deepStrictEqual(parsed.activated_skills, []); + assert.deepStrictEqual(parsed.tool_calls, []); + }), + ); + + it.effect("treats truncated JSONL as malformed and missing result as incomplete", () => + Effect.sync(() => { + const truncated = parseClaudeCliStreamJson( + '{"type":"assistant","message":{"content":[\n', + parseContext, + ); + assert.isTrue(truncated.malformed_jsonl); + assert.isTrue(truncated.incomplete); + + const missingResult = parseClaudeCliStreamJson( + jsonl({ + type: "assistant", + message: { content: [{ type: "text", text: "partial" }] }, + }), + parseContext, + ); + assert.isFalse(missingResult.malformed_jsonl); + assert.isTrue(missingResult.incomplete); + }), + ); + + it.effect("maps sanitized native MCP names and rejects unknown native tools", () => + Effect.sync(() => { + const mapped = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "price_1", + name: NATIVE_PRICE_TOOL, + input: { ids: "ethereum", vs_currencies: "usd" }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [ + { + type: "tool_result", + tool_use_id: "price_1", + content: { ethereum: { usd: 1 } }, + }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.strictEqual(mapped.tool_calls.length, 1); + assert.strictEqual(mapped.tool_calls[0]?.name, "spot.getSimplePrice"); + assert.deepStrictEqual(mapped.tool_calls[0]?.arguments, { + ids: "ethereum", + vs_currencies: "usd", + }); + assert.notInclude( + collectPublicStrings(mapped).join("\n"), + "mcp__ask-gina__spot_getSimplePrice", + ); + assert.notInclude(collectPublicStrings(mapped).join("\n"), NATIVE_PRICE_TOOL); + assert.isFalse(mapped.malformed_jsonl); + assert.isFalse(mapped.incomplete); + assert.deepStrictEqual(mapped.available_tools, listCatalogToolNames()); + + const unknown = parseClaudeCliStreamJson( + jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "unknown_1", + name: "mcp__ask-gina__spot.getSimplePrice", + input: { ids: "ethereum" }, + }, + { + type: "tool_use", + id: "unknown_2", + name: "mcp__ask-gina__not_a_catalog_tool", + input: {}, + }, + ], + }, + }, + { + type: "user", + message: { + content: [ + { type: "tool_result", tool_use_id: "unknown_1", content: {} }, + { type: "tool_result", tool_use_id: "unknown_2", content: {} }, + ], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(unknown.tool_calls, []); + assert.strictEqual(unknown.unsupported_actions, 2); + assert.include(unknown.error ?? "", "outside Skill"); + }), + ); + + it.effect("accepts optional EndConversation init but not inventory drift or execution", () => + Effect.sync(() => { + const initWithEndConversation = { + ...validInit, + tools: [...validInit.tools, "EndConversation"], + }; + const accepted = parseClaudeCliStreamJson( + jsonl(initWithEndConversation, successResult), + parseContext, + ); + assert.deepStrictEqual(accepted.available_tools, listCatalogToolNames()); + assert.isFalse(accepted.malformed_jsonl); + assert.isFalse(accepted.incomplete); + + const rejectedInventories = [ + [...validInit.tools, "UnknownTool"], + validInit.tools.filter((tool) => tool !== NATIVE_PRICE_TOOL), + [...validInit.tools, "EndConversation", "EndConversation"], + ]; + for (const tools of rejectedInventories) { + const rejected = parseClaudeCliStreamJson( + jsonl({ ...validInit, tools }, successResult), + parseContext, + ); + assert.isUndefined(rejected.available_tools); + assert.isFalse(rejected.malformed_jsonl); + assert.isTrue(rejected.incomplete); + } + + const executed = parseClaudeCliStreamJson( + jsonl( + initWithEndConversation, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "end_1", + name: "EndConversation", + input: {}, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "end_1", content: "ended" }], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(executed.tool_calls, []); + assert.strictEqual(executed.unsupported_actions, 1); + assert.include(executed.error ?? "", "outside Skill"); + assert.isFalse(executed.incomplete); + }), + ); + + it.effect("requires a valid init catalog before actions or result", () => + Effect.sync(() => { + const missingInit = parseClaudeCliStreamJson( + jsonl( + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "price_1", + name: NATIVE_PRICE_TOOL, + input: { ids: "ethereum" }, + }, + ], + }, + }, + { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: "price_1", content: {} }], + }, + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(missingInit.tool_calls, []); + assert.isTrue(missingInit.incomplete); + assert.isFalse(missingInit.malformed_jsonl); + assert.isUndefined(missingInit.error); + assert.notInclude(Object.keys(missingInit), "available_tools"); + assert.notInclude(collectPublicStrings(missingInit).join("\n"), PLUGIN_DIRECTORY); + + const pluginError = parseClaudeCliStreamJson( + jsonl( + { + ...validInit, + plugin_errors: [{ name: "ask-gina", error: "failed to load plugin" }], + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(pluginError.tool_calls, []); + assert.isTrue(pluginError.incomplete); + assert.isFalse(pluginError.malformed_jsonl); + assert.isUndefined(pluginError.error); + assert.notInclude(Object.keys(pluginError), "available_tools"); + + const serverError = parseClaudeCliStreamJson( + jsonl( + { + ...validInit, + mcp_server_errors: [{ name: "ask-gina", error: "disconnected" }], + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(serverError.tool_calls, []); + assert.isTrue(serverError.incomplete); + assert.isFalse(serverError.malformed_jsonl); + assert.isUndefined(serverError.error); + assert.notInclude(Object.keys(serverError), "available_tools"); + + const incompleteCatalog = parseClaudeCliStreamJson( + jsonl( + { + ...validInit, + tools: ["Read", "Skill", NATIVE_PRICE_TOOL], + }, + successResult, + ), + parseContext, + ); + assert.deepStrictEqual(incompleteCatalog.tool_calls, []); + assert.isTrue(incompleteCatalog.incomplete); + assert.isFalse(incompleteCatalog.malformed_jsonl); + assert.isUndefined(incompleteCatalog.error); + assert.notInclude(Object.keys(incompleteCatalog), "available_tools"); + }), + ); +}); + +describe("Claude CLI trial adapter", () => { + it.layer(BunServices.layer)((it) => { + it.effect("uses isolated argv and keeps the API key in child env only", () => + Effect.gen(function* () { + let captured: ClaudeCliCommand | undefined; + const runner: ClaudeCliTrialRunner = { + run: (command) => + Effect.sync(() => { + captured = command; + return { + ...fixtureResult, + stdout: jsonl(validInit, successResult), + }; + }), + }; + const parentEnvironment = { + PATH: "/usr/bin:/bin", + HOME: "/home/eval", + LANG: "C.UTF-8", + ANTHROPIC_API_KEY: "parent-secret", + CLAUDE_CODE_MAX_RETRIES: "99", + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: "0", + CLAUDE_CODE_RETRY_WATCHDOG: "1", + AWS_SECRET_ACCESS_KEY: "cloud-secret", + GITHUB_TOKEN: "github-secret", + }; + + const observation = yield* runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment, + runner, + }); + if (captured === undefined) return yield* Effect.die("fixture runner was not invoked"); + + assert.strictEqual(captured.command, trialOptions.executablePath); + assert.include(captured.args, "-p"); + assert.include(captured.args, "--restricted"); + // Bare mode disables native Skill even when --tools explicitly includes it. + assert.notInclude(captured.args, "--bare"); + assert.include(captured.args, "--strict-mcp-config"); + assert.include(captured.args, "--permission-prompts"); + assert.include(captured.args, "none"); + assert.include(captured.args, "--permission-mode"); + assert.include(captured.args, "dontAsk"); + assert.include(captured.args, "--no-session-persistence"); + assert.include(captured.args, "--plugin-dir"); + assert.strictEqual( + captured.args[captured.args.indexOf("--plugin-dir") + 1], + PLUGIN_DIRECTORY, + ); + const allowedTools = captured.args[captured.args.indexOf("--allowedTools") + 1] ?? ""; + assert.notInclude(allowedTools, "mcp__ask-gina__*"); + assert.notInclude(allowedTools, "Skill(ask-gina:*)"); + assert.include(allowedTools, "Skill(ask-gina:research-spot-tokens)"); + assert.include(allowedTools, NATIVE_PRICE_TOOL); + assert.notInclude(allowedTools, "mcp__ask-gina__spot.getSimplePrice"); + assert.include(captured.args, "--mcp-config"); + assert.include(captured.args, "--output-format"); + assert.include(captured.args, "stream-json"); + assert.include(captured.args, "--effort"); + assert.include(captured.args, "medium"); + assert.include(captured.args, "--max-turns"); + assert.include(captured.args, "8"); + assert.notInclude(captured.args, "synthetic-anthropic-key"); + assert.notInclude(captured.args, "gina-read-secret"); + assert.notInclude(captured.args, "parent-secret"); + assert.strictEqual(captured.environment.ANTHROPIC_API_KEY, "synthetic-anthropic-key"); + assert.strictEqual(captured.environment.CLAUDE_CODE_MAX_RETRIES, "0"); + assert.strictEqual(captured.environment.CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK, "1"); + assert.strictEqual(captured.environment.MCP_DISCOVERY_CACHE, "0"); + assert.notInclude(Object.keys(captured.environment), "CLAUDE_CODE_RETRY_WATCHDOG"); + assert.notInclude(Object.keys(captured.environment), "AWS_SECRET_ACCESS_KEY"); + assert.notInclude(Object.keys(captured.environment), "GITHUB_TOKEN"); + assert.strictEqual(observation.target, "claude_cli"); + assert.strictEqual(observation.status, "completed"); + assert.deepStrictEqual(observation.available_tools, listCatalogToolNames()); + assert.notInclude(collectPublicStrings(observation).join("\n"), "synthetic-anthropic-key"); + assert.notInclude(collectPublicStrings(observation).join("\n"), NATIVE_PRICE_TOOL); + }), + ); + + it.effect("terminates CLI options before a flag-like suite prompt", () => + Effect.gen(function* () { + let captured: ClaudeCliCommand | undefined; + const runner: ClaudeCliTrialRunner = { + run: (command) => + Effect.sync(() => { + captured = command; + return { ...fixtureResult, stdout: jsonl(validInit, successResult) }; + }), + }; + const flagLikeCase = { + ...evalCase, + turns: [{ role: "user", content: "--help" }], + } satisfies PluginEvalCase; + + yield* runClaudeCliPluginEvalTrial(flagLikeCase, { + ...trialOptions, + parentEnvironment: {}, + runner, + }); + if (captured === undefined) return yield* Effect.die("fixture runner was not invoked"); + + assert.deepStrictEqual(captured.args.slice(-2), ["--", "--help"]); + }), + ); + + it.effect("returns typed spawn and timeout failures without child payload text", () => + Effect.gen(function* () { + const spawnRunner: ClaudeCliTrialRunner = { + run: (command) => + Effect.fail( + new PluginEvalClaudeCliSpawnError({ + caseId: command.caseId, + reason: "could_not_start", + }), + ), + }; + const spawnResult = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: { ANTHROPIC_API_KEY: "must-not-leak" }, + runner: spawnRunner, + }), + ); + assert.strictEqual(spawnResult._tag, "Failure"); + if (spawnResult._tag === "Failure") { + assert.instanceOf(spawnResult.failure, PluginEvalClaudeCliSpawnError); + assert.notInclude(collectPublicStrings(spawnResult.failure).join("\n"), "must-not-leak"); + assert.notInclude( + collectPublicStrings(spawnResult.failure).join("\n"), + "synthetic-anthropic-key", + ); + } + + let interrupted = false; + const timeoutRunner: ClaudeCliTrialRunner = { + run: () => + Effect.never.pipe( + Effect.ensuring( + Effect.sync(() => { + interrupted = true; + }), + ), + ), + }; + const timeoutResult = yield* TestClock.withLive( + Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: {}, + timeoutMs: 1, + runner: timeoutRunner, + }), + ), + ); + assert.strictEqual(timeoutResult._tag, "Failure"); + if (timeoutResult._tag === "Failure") { + assert.instanceOf(timeoutResult.failure, PluginEvalClaudeCliTimeoutError); + assert.isTrue(interrupted); + if (timeoutResult.failure._tag === "PluginEvalClaudeCliTimeoutError") { + assert.deepStrictEqual( + { + caseId: timeoutResult.failure.caseId, + timeoutMs: timeoutResult.failure.timeoutMs, + }, + { caseId: evalCase.id, timeoutMs: 1 }, + ); + } + assert.notInclude( + collectPublicStrings(timeoutResult.failure).join("\n"), + "ANTHROPIC_API_KEY", + ); + } + }), + ); + + it.effect( + "fails typed invalid-process evidence while scoring unsupported completed actions", + () => + Effect.gen(function* () { + const scenarios = [ + { + reason: "stdout-truncated" as const, + result: { ...fixtureResult, stdoutTruncated: true }, + }, + { + reason: "stderr-truncated" as const, + result: { ...fixtureResult, stderrTruncated: true }, + }, + { + reason: "nonzero-exit" as const, + result: { ...fixtureResult, exitCode: 7 }, + }, + { + reason: "malformed-jsonl" as const, + result: { ...fixtureResult, stdout: '{"type":"assistant","message":{"content":[\n' }, + }, + { + reason: "incomplete-stream" as const, + result: { + ...fixtureResult, + stdout: jsonl({ + type: "assistant", + message: { content: [{ type: "text", text: "cut off" }] }, + }), + }, + }, + ]; + for (const scenario of scenarios) { + const result = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: {}, + runner: { run: () => Effect.succeed(scenario.result) }, + }), + ); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalClaudeCliProcessError); + if (result.failure instanceof PluginEvalClaudeCliProcessError) { + assert.strictEqual(result.failure.reason, scenario.reason); + assert.notInclude(collectPublicStrings(result.failure).join("\n"), "cut off"); + assert.notInclude( + collectPublicStrings(result.failure).join("\n"), + "curl example.com", + ); + } + } + } + + const unsupported = yield* runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: {}, + runner: { + run: () => + Effect.succeed({ + ...fixtureResult, + stdout: jsonl( + validInit, + { + type: "assistant", + message: { + content: [ + { + type: "tool_use", + id: "bash_1", + name: "Bash", + input: { command: "curl example.com" }, + }, + ], + }, + }, + successResult, + ), + }), + }, + }); + assert.strictEqual(unsupported.status, "failed"); + assert.include(unsupported.error ?? "", "outside Skill"); + + const missingInitTrial = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: {}, + runner: { + run: () => + Effect.succeed({ + ...fixtureResult, + stdout: jsonl(successResult), + }), + }, + }), + ); + assert.strictEqual(missingInitTrial._tag, "Failure"); + if (missingInitTrial._tag === "Failure") { + assert.instanceOf(missingInitTrial.failure, PluginEvalClaudeCliProcessError); + if (missingInitTrial.failure instanceof PluginEvalClaudeCliProcessError) { + assert.strictEqual(missingInitTrial.failure.reason, "incomplete-stream"); + } + } + + const pluginLoadTrial = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + parentEnvironment: {}, + runner: { + run: () => + Effect.succeed({ + ...fixtureResult, + stdout: jsonl( + { + ...validInit, + plugin_errors: [{ name: "ask-gina", error: "failed to load plugin" }], + }, + successResult, + ), + }), + }, + }), + ); + assert.strictEqual(pluginLoadTrial._tag, "Failure"); + if (pluginLoadTrial._tag === "Failure") { + assert.instanceOf(pluginLoadTrial.failure, PluginEvalClaudeCliProcessError); + if (pluginLoadTrial.failure instanceof PluginEvalClaudeCliProcessError) { + assert.strictEqual(pluginLoadTrial.failure.reason, "incomplete-stream"); + assert.notInclude( + collectPublicStrings(pluginLoadTrial.failure).join("\n"), + "failed to load plugin", + ); + } + } + }), + ); + + it.effect( + "rejects noncanonical catalog evidence and invalid options before invoking the runner", + () => + Effect.gen(function* () { + let invoked = false; + const runner: ClaudeCliTrialRunner = { + run: () => { + invoked = true; + return Effect.succeed(fixtureResult); + }, + }; + const catalogResult = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + availableTools: listCatalogToolNames().slice(1), + parentEnvironment: {}, + runner, + }), + ); + assert.isFalse(invoked); + assert.strictEqual(catalogResult._tag, "Failure"); + if (catalogResult._tag === "Failure") { + assert.instanceOf(catalogResult.failure, PluginEvalClaudeCliSpawnError); + if (catalogResult.failure instanceof PluginEvalClaudeCliSpawnError) { + assert.strictEqual(catalogResult.failure.reason, "catalog-mismatch"); + } + } + + const maxTurnsResult = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + maxTurns: 33, + parentEnvironment: {}, + runner, + }), + ); + assert.isFalse(invoked); + assert.strictEqual(maxTurnsResult._tag, "Failure"); + if (maxTurnsResult._tag === "Failure") { + assert.instanceOf(maxTurnsResult.failure, PluginEvalClaudeCliSpawnError); + if (maxTurnsResult.failure instanceof PluginEvalClaudeCliSpawnError) { + assert.strictEqual(maxTurnsResult.failure.reason, "invalid-options"); + } + } + + const reasoningResult = yield* Effect.result( + runClaudeCliPluginEvalTrial(evalCase, { + ...trialOptions, + reasoning: "turbo", + parentEnvironment: {}, + runner, + }), + ); + assert.isFalse(invoked); + assert.strictEqual(reasoningResult._tag, "Failure"); + if (reasoningResult._tag === "Failure") { + assert.instanceOf(reasoningResult.failure, PluginEvalClaudeCliSpawnError); + assert.notInclude(collectPublicStrings(reasoningResult.failure).join("\n"), "turbo"); + } + }), + ); + }); +}); + +describe("Claude CLI environment allowlist", () => { + it.effect("drops provider, cloud, registry, GitHub, SSH, proxy, and repository secrets", () => + Effect.sync(() => { + const environment = buildClaudeCliEnvironment({ + PATH: "/bin", + HOME: "/home/eval", + SystemRoot: "C:\\Windows", + LC_ALL: "C", + TEMP: "C:\\Temp", + ANTHROPIC_API_KEY: "provider", + GOOGLE_APPLICATION_CREDENTIALS: "cloud", + NPM_CONFIG_TOKEN: "registry", + GH_TOKEN: "github", + SSH_PRIVATE_KEY: "ssh", + ALL_PROXY: "proxy", + REPOSITORY_SECRET: "repo", + }); + + assert.deepStrictEqual(environment, { + PATH: "/bin", + HOME: "/home/eval", + SystemRoot: "C:\\Windows", + LC_ALL: "C", + TEMP: "C:\\Temp", + }); + }), + ); +}); diff --git a/packages/evals/__tests__/live-cli.test.ts b/packages/evals/__tests__/live-cli.test.ts new file mode 100644 index 0000000..9a8b7b2 --- /dev/null +++ b/packages/evals/__tests__/live-cli.test.ts @@ -0,0 +1,396 @@ +import * as BunPath from "@effect/platform-bun/BunPath"; +import * as BunServices from "@effect/platform-bun/BunServices"; +import { assert, describe, it } from "@effect/vitest"; +import { Config, ConfigProvider, Effect } from "effect"; +import { ChildProcess } from "effect/unstable/process"; + +import { collectBoundedUtf8Output } from "../src/bounded-output"; + +import { + DEFAULT_CLAUDE_MAX_TURNS, + DEFAULT_OPENROUTER_MAX_STEPS, + formatLiveEvalCliFailure, + formatLiveEvalCliUsage, + loadLiveEvalCredentials, + parseLiveEvalCliOptions, +} from "../src/bin/live"; + +const requiredFlags = (runner: string, extra: readonly string[] = []): readonly string[] => [ + "--runner", + runner, + "--suite", + "suite.yaml", + "--run-id", + "run-1", + "--candidate", + "cand-1", + "--model", + "test-model", + "--reasoning", + "medium", + "--repetitions", + "3", + "--account-class", + "local", + "--timeout-ms", + "120000", + ...extra, +]; + +const withEnv = + (env: Record) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnv({ env })), + ); + +describe("live eval CLI parser", () => { + it.effect("parses supported runners and preserves existing required flags", () => + Effect.gen(function* () { + const responses = yield* parseLiveEvalCliOptions(requiredFlags("responses")); + assert.strictEqual(responses.mode, "run"); + if (responses.mode === "run") { + assert.strictEqual(responses.options.runner, "responses"); + assert.strictEqual(responses.options.model, "test-model"); + assert.strictEqual(responses.options.reasoning, "medium"); + assert.strictEqual(responses.options.repetitions, 3); + assert.notProperty(responses.options, "maxSteps"); + assert.notProperty(responses.options, "maxTurns"); + } + + const openrouter = yield* parseLiveEvalCliOptions(requiredFlags("openrouter")); + assert.strictEqual(openrouter.mode, "run"); + if (openrouter.mode === "run" && openrouter.options.runner === "openrouter") { + assert.strictEqual(openrouter.options.maxSteps, DEFAULT_OPENROUTER_MAX_STEPS); + } + + const claude = yield* parseLiveEvalCliOptions(requiredFlags("claude")); + assert.strictEqual(claude.mode, "run"); + if (claude.mode === "run" && claude.options.runner === "claude") { + assert.strictEqual(claude.options.maxTurns, DEFAULT_CLAUDE_MAX_TURNS); + } + + const codex = yield* parseLiveEvalCliOptions( + requiredFlags("codex", [ + "--attempts-output", + "attempts.json", + "--case", + "list-scheduled-prompts", + ]), + ); + assert.strictEqual(codex.mode, "run"); + if (codex.mode === "run") { + assert.strictEqual(codex.options.runner, "codex"); + assert.strictEqual(codex.options.attemptsOutputPath, "attempts.json"); + assert.deepStrictEqual(codex.options.caseIds, ["list-scheduled-prompts"]); + } + + const omp = yield* parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "anthropic"])); + assert.strictEqual(omp.mode, "run"); + if (omp.mode === "run" && omp.options.runner === "omp") { + assert.strictEqual(omp.options.provider, "anthropic"); + assert.strictEqual(omp.options.model, "test-model"); + assert.notProperty(omp.options, "maxSteps"); + assert.notProperty(omp.options, "maxTurns"); + } + + const missingProvider = yield* Effect.result(parseLiveEvalCliOptions(requiredFlags("omp"))); + assert.strictEqual(missingProvider._tag, "Failure"); + const unknownProvider = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "google"])), + ); + assert.strictEqual(unknownProvider._tag, "Failure"); + }), + ); + + it.effect("accepts bounded OpenRouter and Claude budgets", () => + Effect.gen(function* () { + const steps = yield* parseLiveEvalCliOptions( + requiredFlags("openrouter", ["--max-steps", "32"]), + ); + assert.strictEqual(steps.mode, "run"); + if (steps.mode === "run" && steps.options.runner === "openrouter") { + assert.strictEqual(steps.options.maxSteps, 32); + } + const turns = yield* parseLiveEvalCliOptions(requiredFlags("claude", ["--max-turns", "1"])); + assert.strictEqual(turns.mode, "run"); + if (turns.mode === "run" && turns.options.runner === "claude") { + assert.strictEqual(turns.options.maxTurns, 1); + } + }), + ); + + it.effect("rejects irrelevant budgets and values outside 1..32", () => + Effect.gen(function* () { + const responsesSteps = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("responses", ["--max-steps", "8"])), + ); + assert.strictEqual(responsesSteps._tag, "Failure"); + const codexTurns = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("codex", ["--max-turns", "8"])), + ); + assert.strictEqual(codexTurns._tag, "Failure"); + const openrouterTurns = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("openrouter", ["--max-turns", "8"])), + ); + assert.strictEqual(openrouterTurns._tag, "Failure"); + const claudeSteps = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("claude", ["--max-steps", "8"])), + ); + assert.strictEqual(claudeSteps._tag, "Failure"); + const tooHigh = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("openrouter", ["--max-steps", "33"])), + ); + assert.strictEqual(tooHigh._tag, "Failure"); + const zero = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("claude", ["--max-turns", "0"])), + ); + assert.strictEqual(zero._tag, "Failure"); + const responsesProvider = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("responses", ["--provider", "openai"])), + ); + assert.strictEqual(responsesProvider._tag, "Failure"); + const ompSteps = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "openai", "--max-steps", "8"])), + ); + assert.strictEqual(ompSteps._tag, "Failure"); + }), + ); + + it.effect("still requires model, reasoning, repetitions, account class, and timeout", () => + Effect.gen(function* () { + const missingModel = yield* Effect.result( + parseLiveEvalCliOptions([ + "--runner", + "openrouter", + "--suite", + "suite.yaml", + "--run-id", + "run-1", + "--candidate", + "cand-1", + "--reasoning", + "medium", + "--repetitions", + "3", + "--account-class", + "local", + "--timeout-ms", + "120000", + ]), + ); + assert.strictEqual(missingModel._tag, "Failure"); + if (missingModel._tag === "Failure") { + assert.strictEqual(missingModel.failure.reason, "invalid-arguments"); + assert.include(formatLiveEvalCliFailure(missingModel.failure), formatLiveEvalCliUsage()); + } + }), + ); +}); + +describe("live eval CLI credentials", () => { + it.layer(BunPath.layer)((it) => { + it.effect("loads only the selected OpenRouter key and names missing variables", () => + Effect.gen(function* () { + const missingGina = yield* loadLiveEvalCredentials("openrouter").pipe( + withEnv({}), + Effect.result, + ); + assert.strictEqual(missingGina._tag, "Failure"); + if (missingGina._tag === "Failure") { + assert.strictEqual(missingGina.failure.reason, "invalid-credentials"); + assert.deepStrictEqual(missingGina.failure.missing, ["ASK_GINA_ACCESS_TOKEN"]); + const message = formatLiveEvalCliFailure(missingGina.failure); + assert.include(message, "ASK_GINA_ACCESS_TOKEN"); + assert.notInclude(message, "/"); + } + + const missingProvider = yield* loadLiveEvalCredentials("openrouter").pipe( + withEnv({ ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token" }), + Effect.result, + ); + assert.strictEqual(missingProvider._tag, "Failure"); + if (missingProvider._tag === "Failure") { + assert.deepStrictEqual(missingProvider.failure.missing, ["OPENROUTER_API_KEY"]); + assert.notInclude( + formatLiveEvalCliFailure(missingProvider.failure), + "synthetic-gina-token", + ); + } + + const loaded = yield* loadLiveEvalCredentials("openrouter").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OPENROUTER_API_KEY: "synthetic-openrouter-key", + OPENAI_API_KEY: "must-not-be-required", + }), + ); + assert.strictEqual(loaded.runner, "openrouter"); + }), + ); + + it.effect("does not require OPENAI_API_KEY for Claude and hides executable paths", () => + Effect.gen(function* () { + const relative = yield* loadLiveEvalCredentials("claude").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + ANTHROPIC_API_KEY: "synthetic-anthropic-key", + CLAUDE_EVAL_EXECUTABLE: "relative/claude", + }), + Effect.result, + ); + assert.strictEqual(relative._tag, "Failure"); + if (relative._tag === "Failure") { + assert.deepStrictEqual(relative.failure.missing, ["CLAUDE_EVAL_EXECUTABLE"]); + const message = formatLiveEvalCliFailure(relative.failure); + assert.include(message, "CLAUDE_EVAL_EXECUTABLE"); + assert.notInclude(message, "relative/claude"); + } + + const loaded = yield* loadLiveEvalCredentials("claude").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + ANTHROPIC_API_KEY: "synthetic-anthropic-key", + CLAUDE_EVAL_EXECUTABLE: "/usr/bin/claude", + }), + ); + assert.strictEqual(loaded.runner, "claude"); + if (loaded.runner === "claude") { + assert.strictEqual(loaded.executablePath, "/usr/bin/claude"); + } + }), + ); + + it.effect("still requires OPENAI_API_KEY only for Responses and Codex", () => + Effect.gen(function* () { + const responses = yield* loadLiveEvalCredentials("responses").pipe( + withEnv({ ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token" }), + Effect.result, + ); + assert.strictEqual(responses._tag, "Failure"); + if (responses._tag === "Failure") { + assert.deepStrictEqual(responses.failure.missing, ["OPENAI_API_KEY"]); + } + + const loadedResponses = yield* loadLiveEvalCredentials("responses").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OPENAI_API_KEY: "synthetic-openai-key", + }), + ); + assert.strictEqual(loadedResponses.runner, "responses"); + + const missingCodexExecutable = yield* loadLiveEvalCredentials("codex").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OPENAI_API_KEY: "synthetic-openai-key", + }), + Effect.result, + ); + assert.strictEqual(missingCodexExecutable._tag, "Failure"); + if (missingCodexExecutable._tag === "Failure") { + assert.deepStrictEqual(missingCodexExecutable.failure.missing, ["CODEX_EVAL_EXECUTABLE"]); + assert.notInclude( + formatLiveEvalCliFailure(missingCodexExecutable.failure), + "synthetic-openai-key", + ); + } + }), + ); + + it.effect("loads OMP pins without OpenAI keys and hides executable paths", () => + Effect.gen(function* () { + const missingKey = yield* loadLiveEvalCredentials("omp").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OPENAI_API_KEY: "must-not-be-required", + }), + Effect.result, + ); + assert.strictEqual(missingKey._tag, "Failure"); + if (missingKey._tag === "Failure") { + assert.deepStrictEqual(missingKey.failure.missing, ["OMP_EVAL_API_KEY"]); + assert.notInclude(formatLiveEvalCliFailure(missingKey.failure), "must-not-be-required"); + } + + const relative = yield* loadLiveEvalCredentials("omp").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OMP_EVAL_API_KEY: "synthetic-omp-key", + OMP_EVAL_EXECUTABLE: "relative/omp", + OMP_EVAL_EXECUTABLE_SHA256: "abc", + }), + Effect.result, + ); + assert.strictEqual(relative._tag, "Failure"); + if (relative._tag === "Failure") { + assert.deepStrictEqual(relative.failure.missing, ["OMP_EVAL_EXECUTABLE"]); + assert.notInclude(formatLiveEvalCliFailure(relative.failure), "relative/omp"); + } + + const loaded = yield* loadLiveEvalCredentials("omp").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OMP_EVAL_API_KEY: "synthetic-omp-key", + OMP_EVAL_EXECUTABLE: "/usr/bin/omp", + OMP_EVAL_EXECUTABLE_SHA256: "AbCDEF", + OPENAI_API_KEY: "must-not-be-required", + }), + ); + assert.strictEqual(loaded.runner, "omp"); + if (loaded.runner === "omp") { + assert.strictEqual(loaded.executablePath, "/usr/bin/omp"); + assert.strictEqual(loaded.expectedSha256, "abcdef"); + } + + const responses = yield* loadLiveEvalCredentials("responses").pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OPENAI_API_KEY: "synthetic-openai-key", + OMP_EVAL_API_KEY: "must-not-be-required", + }), + ); + assert.strictEqual(responses.runner, "responses"); + }), + ); + }); +}); + +describe("live eval CLI subprocess", () => { + it.layer(BunServices.layer)((it) => { + it.effect("prints help without credentials or network", () => + Effect.scoped( + Effect.gen(function* () { + const pathValue = yield* Config.string("PATH"); + const child = yield* ChildProcess.make( + "bun", + ["packages/evals/src/bin/live.ts", "--help"], + { + cwd: process.cwd(), + env: { PATH: pathValue }, + extendEnv: false, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + collectBoundedUtf8Output(child.stdout, 65_536), + collectBoundedUtf8Output(child.stderr, 65_536), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + assert.strictEqual(exitCode, 0); + assert.include(stdout.text, "eval:"); + assert.include(stdout.text, "ASK_GINA_ACCESS_TOKEN"); + assert.include(stdout.text, "OMP_EVAL_API_KEY"); + assert.notInclude(stdout.text, "sk-"); + assert.notInclude(stderr.text, "OPENAI_API_KEY="); + assert.notInclude(stderr.text, "OMP_EVAL_API_KEY="); + }), + ), + ); + }); +}); diff --git a/packages/evals/__tests__/openrouter.test.ts b/packages/evals/__tests__/openrouter.test.ts new file mode 100644 index 0000000..d7fe7e0 --- /dev/null +++ b/packages/evals/__tests__/openrouter.test.ts @@ -0,0 +1,638 @@ +import { createMCPClient } from "@ai-sdk/mcp"; +import { listCatalogToolNames, PRODUCTION_MCP_URL } from "@askgina/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import { generateText } from "ai"; +import { Cause, Clock, Effect, Exit, Fiber } from "effect"; +import { beforeEach, vi } from "vitest"; + +import type { PluginEvalCase } from "../src/contracts"; +import { + PluginEvalOpenRouterGenerationError, + PluginEvalOpenRouterMcpError, + PluginEvalOpenRouterRequestError, + PluginEvalOpenRouterTimeoutError, + type OpenRouterTrialOptions, + runOpenRouterPluginEvalTrial, +} from "../src/openrouter"; + +vi.mock("@ai-sdk/mcp", () => ({ + createMCPClient: vi.fn(), +})); + +vi.mock("@openrouter/ai-sdk-provider", () => ({ + createOpenRouter: vi.fn(() => (modelId: string) => ({ modelId })), +})); + +vi.mock("ai", () => ({ + generateText: vi.fn(), + isStepCount: (count: number) => count, +})); + +const evalCase: PluginEvalCase = { + id: "direct-price", + category: "direct", + tags: ["spot"], + manual_priority: "required", + turns: [{ role: "user", content: "Show Ethereum in USD" }], + expected: { + routing: { kind: "exact", tool: "spot.getSimplePrice" }, + }, +}; + +const allowedTools = listCatalogToolNames(); +const createMCPClientMock = vi.mocked(createMCPClient); +const generateTextMock = vi.mocked(generateText); + +const options = { + apiKey: "synthetic-openrouter-key", + mcpAuthorization: "gina-read-secret", + model: "openai/gpt-4o", + reasoning: "medium", + runId: "run-1", + repetition: 1, + serverUrl: PRODUCTION_MCP_URL, + allowedTools, +} as const satisfies OpenRouterTrialOptions; + +const serialized = (value: unknown): string => JSON.stringify(value); + +const catalogTools = (names: readonly string[], nextCursor?: string) => ({ + tools: names.map((name) => ({ name, inputSchema: { type: "object" } })), + ...(nextCursor === undefined ? {} : { nextCursor }), +}); + +const mockClient = (overrides?: { + readonly tools?: readonly string[]; + readonly close?: () => Promise; +}) => ({ + listTools: vi.fn(() => Promise.resolve(catalogTools(overrides?.tools ?? allowedTools))), + toolsFromDefinitions: vi.fn((definitions: { tools: readonly { name: string }[] }) => + Object.fromEntries(definitions.tools.map((tool) => [tool.name, tool])), + ), + close: vi.fn(overrides?.close ?? (() => Promise.resolve())), +}); + +describe("OpenRouter trial adapter", () => { + beforeEach(() => { + createMCPClientMock.mockReset(); + generateTextMock.mockReset(); + }); + + it.effect("rejects non-canonical options and unknown reasoning before connecting", () => + Effect.gen(function* () { + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { + ...options, + serverUrl: "https://example.invalid/mcp", + }), + ); + + assert.strictEqual(createMCPClientMock.mock.calls.length, 0); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterRequestError); + assert.strictEqual(result.failure.reason, "invalid-options"); + assert.notInclude(serialized(result.failure), "example.invalid"); + assert.notInclude(serialized(result.failure), options.apiKey); + } + + const reasoningResult = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, reasoning: "turbo" }), + ); + assert.strictEqual(createMCPClientMock.mock.calls.length, 0); + assert.strictEqual(reasoningResult._tag, "Failure"); + if (reasoningResult._tag === "Failure") { + assert.instanceOf(reasoningResult.failure, PluginEvalOpenRouterRequestError); + assert.strictEqual(reasoningResult.failure.reason, "unsupported-reasoning"); + assert.notInclude(serialized(reasoningResult.failure), "turbo"); + } + + const emptyReasoning = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, reasoning: "" }), + ); + assert.strictEqual(emptyReasoning._tag, "Failure"); + if (emptyReasoning._tag === "Failure") { + assert.instanceOf(emptyReasoning.failure, PluginEvalOpenRouterRequestError); + assert.strictEqual(emptyReasoning.failure.reason, "unsupported-reasoning"); + } + + const maxStepsResult = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, maxSteps: 33 }), + ); + assert.strictEqual(maxStepsResult._tag, "Failure"); + if (maxStepsResult._tag === "Failure") { + assert.instanceOf(maxStepsResult.failure, PluginEvalOpenRouterRequestError); + assert.strictEqual(maxStepsResult.failure.reason, "invalid-options"); + } + + const subsetResult = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { + ...options, + allowedTools: allowedTools.slice(0, -1), + }), + ); + assert.strictEqual(createMCPClientMock.mock.calls.length, 0); + assert.strictEqual(subsetResult._tag, "Failure"); + if (subsetResult._tag === "Failure") { + assert.instanceOf(subsetResult.failure, PluginEvalOpenRouterRequestError); + assert.strictEqual(subsetResult.failure.reason, "invalid-options"); + } + }), + ); + + it.effect("rejects a discovered catalog that differs from allowed_tools and closes MCP", () => + Effect.gen(function* () { + const providerCatalogValue = "provider-catalog-value-must-not-be-copied"; + const client = mockClient({ tools: [providerCatalogValue] }); + createMCPClientMock.mockResolvedValue(client as never); + + const result = yield* Effect.result(runOpenRouterPluginEvalTrial(evalCase, options)); + + assert.strictEqual(generateTextMock.mock.calls.length, 0); + assert.strictEqual(client.toolsFromDefinitions.mock.calls.length, 0); + assert.strictEqual(client.close.mock.calls.length, 1); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterMcpError); + assert.strictEqual(result.failure.reason, "catalog-mismatch"); + assert.notInclude(serialized(result.failure), providerCatalogValue); + assert.notInclude(serialized(result.failure), options.apiKey); + assert.notInclude(serialized(result.failure), options.mcpAuthorization); + } + }), + ); + + it.effect("records ordered tool calls and omits incomplete usage and secret values", () => + Effect.gen(function* () { + const toolOutput = '{"ethereum":{"usd":3200}}'; + const providerToolError = "provider-call-message-must-not-be-copied"; + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockResolvedValue({ + text: "ETH is $3,200.", + finishReason: "stop", + usage: { + inputTokens: 110, + outputTokens: 20, + totalTokens: 130, + }, + steps: [ + { + usage: { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + }, + toolCalls: [ + { + type: "tool-call", + toolCallId: "call_1", + toolName: "spot_getSimplePrice", + input: { ids: "ethereum", vs_currencies: "usd" }, + }, + { + type: "tool-call", + toolCallId: "call_2", + toolName: "gina_getCrosschainPortfolio", + input: "{invalid-provider-arguments", + }, + { + type: "tool-call", + toolCallId: "call_3", + toolName: "spot_getSimplePrice", + input: { ids: "bitcoin" }, + }, + ], + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "spot_getSimplePrice", + output: toolOutput, + }, + { + type: "tool-result", + toolCallId: "call_3", + toolName: "spot_getSimplePrice", + output: { + content: [{ type: "text", text: providerToolError }], + isError: true, + }, + }, + ], + performance: { toolExecutionMs: { call_1: 7, call_3: 4 } }, + }, + ], + } as never); + + const observation = yield* runOpenRouterPluginEvalTrial(evalCase, options); + + assert.strictEqual(observation.status, "completed"); + assert.strictEqual(observation.error, undefined); + assert.strictEqual(observation.target, "openrouter_api"); + assert.strictEqual(observation.final_answer, "ETH is $3,200."); + assert.deepStrictEqual(observation.available_tools, allowedTools); + assert.deepStrictEqual(observation.token_usage, { + input_tokens: 110, + output_tokens: 20, + total_tokens: 130, + }); + assert.strictEqual(observation.activated_skills, undefined); + assert.deepStrictEqual(observation.tool_calls[0], { + sequence: 0, + name: "spot.getSimplePrice", + arguments: { ids: "ethereum", vs_currencies: "usd" }, + duration_ms: 7, + result_bytes: new TextEncoder().encode(toolOutput).byteLength, + }); + assert.deepStrictEqual(observation.tool_calls[1], { + sequence: 1, + name: "gina.getCrosschainPortfolio", + arguments: {}, + error: { + code: "invalid_arguments", + message: "OpenRouter returned invalid MCP arguments", + }, + }); + assert.strictEqual(observation.tool_calls[2]?.sequence, 2); + assert.strictEqual(observation.tool_calls[2]?.name, "spot.getSimplePrice"); + assert.deepStrictEqual(observation.tool_calls[2]?.arguments, { ids: "bitcoin" }); + assert.strictEqual(observation.tool_calls[2]?.duration_ms, 4); + assert.isAtLeast(observation.tool_calls[2]?.result_bytes ?? 0, 1); + assert.deepStrictEqual(observation.tool_calls[2]?.error, { message: "MCP tool call failed" }); + assert.notInclude(serialized(observation), providerToolError); + assert.notInclude(serialized(observation), options.apiKey); + assert.notInclude(serialized(observation), options.mcpAuthorization); + assert.strictEqual(client.close.mock.calls.length, 1); + const mcpConfig = createMCPClientMock.mock.calls[0]?.[0] as { maxRetries?: number }; + const generateConfig = generateTextMock.mock.calls[0]?.[0] as { + maxRetries?: number; + stopWhen?: unknown; + }; + assert.strictEqual(mcpConfig.maxRetries, 0); + assert.strictEqual(generateConfig.maxRetries, 0); + assert.strictEqual(generateConfig.stopWhen, 8); + }), + ); + + it.effect("keeps OpenRouter wire names legal without changing MCP dispatch or reports", () => + Effect.gen(function* () { + const dispatched: string[] = []; + const client = mockClient(); + client.toolsFromDefinitions.mockImplementation( + (definitions: { tools: readonly { name: string }[] }) => + Object.fromEntries( + definitions.tools.map((tool) => [ + tool.name, + { + name: tool.name, + execute: () => { + dispatched.push(tool.name); + return { ok: true }; + }, + }, + ]), + ), + ); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockImplementation(((config: { + tools?: Record unknown }>; + }) => { + config.tools?.spot_getSimplePrice?.execute?.({ ids: "ethereum" }); + return Promise.resolve({ + text: "ETH is $3,200.", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + steps: [ + { + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + toolCalls: [ + { + type: "tool-call", + toolCallId: "call_1", + toolName: "spot_getSimplePrice", + input: { ids: "ethereum" }, + }, + { + type: "tool-call", + toolCallId: "call_unknown", + toolName: "not_a_gina_tool", + input: {}, + }, + ], + content: [ + { + type: "tool-result", + toolCallId: "call_1", + toolName: "spot_getSimplePrice", + output: { ok: true }, + }, + ], + performance: { toolExecutionMs: { call_1: 3 } }, + }, + ], + }); + }) as never); + + const observation = yield* runOpenRouterPluginEvalTrial(evalCase, options); + const generateConfig = generateTextMock.mock.calls[0]?.[0] as { + tools?: Record; + toolOrder?: readonly string[]; + }; + const wireNames = Object.keys(generateConfig.tools ?? {}); + const wireOrder = generateConfig.toolOrder ?? []; + const illegalWireNames = [...wireNames, ...wireOrder].filter( + (name) => !/^[A-Za-z0-9_-]{1,64}$/.test(name), + ); + + assert.deepStrictEqual(illegalWireNames, []); + assert.notInclude(wireNames, "spot.getSimplePrice"); + assert.notInclude(wireOrder, "spot.getSimplePrice"); + assert.include(wireNames, "spot_getSimplePrice"); + assert.include(wireOrder, "spot_getSimplePrice"); + assert.deepStrictEqual(dispatched, ["spot.getSimplePrice"]); + assert.deepStrictEqual(observation.available_tools, allowedTools); + assert.strictEqual(observation.tool_calls[0]?.name, "spot.getSimplePrice"); + assert.deepStrictEqual(observation.tool_calls[0]?.arguments, { ids: "ethereum" }); + assert.deepStrictEqual(observation.tool_calls[1], { + sequence: 1, + name: "not_a_gina_tool", + arguments: {}, + error: { + code: "invalid_tool", + message: "OpenRouter requested an unavailable MCP tool", + }, + }); + assert.notInclude(serialized(observation.tool_calls[0]), "spot_getSimplePrice"); + }), + ); + + it.effect("does not mark step-budget exhaustion as completed", () => + Effect.gen(function* () { + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockResolvedValue({ + text: "partial answer after tool loop", + finishReason: "tool-calls", + usage: { + inputTokens: 80, + outputTokens: 30, + totalTokens: 110, + }, + steps: [], + } as never); + + const observation = yield* runOpenRouterPluginEvalTrial(evalCase, options); + assert.strictEqual(observation.status, "failed"); + assert.strictEqual( + observation.error, + "OpenRouter generation did not complete with a final answer", + ); + assert.strictEqual(observation.final_answer, "partial answer after tool loop"); + assert.deepStrictEqual(observation.token_usage, { + input_tokens: 80, + output_tokens: 30, + total_tokens: 110, + }); + }), + ); + + it.effect("omits token usage unless every required count is present", () => + Effect.gen(function* () { + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockResolvedValue({ + text: "", + finishReason: "stop", + usage: { + inputTokens: 110, + outputTokens: undefined, + totalTokens: 110, + }, + steps: [], + } as never); + + const observation = yield* runOpenRouterPluginEvalTrial(evalCase, options); + assert.strictEqual(observation.status, "failed"); + assert.strictEqual( + observation.error, + "OpenRouter generation did not complete with a final answer", + ); + assert.strictEqual(observation.token_usage, undefined); + assert.strictEqual(observation.final_answer, undefined); + assert.deepStrictEqual(observation.tool_calls, []); + }), + ); + + it.effect("maps generation failures without copying provider messages", () => + Effect.gen(function* () { + const providerValue = "openrouter-provider-message-must-not-be-copied"; + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockImplementation(() => { + throw new Error(providerValue); + }); + + const result = yield* Effect.result(runOpenRouterPluginEvalTrial(evalCase, options)); + assert.strictEqual(client.close.mock.calls.length, 1); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterGenerationError); + assert.strictEqual(result.failure.reason, "generation-failed"); + assert.notInclude(serialized(result.failure), providerValue); + assert.notInclude(serialized(result.failure), options.apiKey); + assert.notInclude(serialized(result.failure), options.mcpAuthorization); + } + }), + ); + + it.live("interrupts a hanging generation, closes MCP, and returns the typed timeout", () => + Effect.gen(function* () { + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockImplementation(({ abortSignal }: { abortSignal?: AbortSignal }) => { + const { promise, reject } = Promise.withResolvers(); + abortSignal?.addEventListener("abort", () => reject(abortSignal.reason), { once: true }); + return promise; + }); + + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, timeoutMs: 1 }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterTimeoutError); + assert.deepStrictEqual( + { caseId: result.failure.caseId, timeoutMs: result.failure.timeoutMs }, + { caseId: evalCase.id, timeoutMs: 1 }, + ); + assert.notInclude(serialized(result.failure), options.apiKey); + assert.notInclude(serialized(result.failure), options.mcpAuthorization); + } + assert.strictEqual(client.close.mock.calls.length, 1); + }), + ); + + it.effect("treats a generation that settles after the deadline as timeout", () => + Effect.gen(function* () { + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + let now = 0; + generateTextMock.mockImplementation((() => { + now = 50; + return Promise.resolve({ + text: "late success must not be recorded", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + steps: [], + }); + }) as never); + const testClock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + monotonicTimeNanosUnsafe: () => 0n, + monotonicTimeNanos: Effect.succeed(0n), + currentTimeNanosUnsafe: () => 0n, + currentTimeNanos: Effect.succeed(0n), + sleep: () => Effect.never, + }; + + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, timeoutMs: 10 }).pipe( + Effect.provideService(Clock.Clock, testClock), + ), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterTimeoutError); + assert.strictEqual(result.failure.timeoutMs, 10); + assert.notInclude(serialized(result.failure), "late success must not be recorded"); + } + assert.strictEqual(client.close.mock.calls.length, 1); + }), + ); + + it.effect("treats a generation that rejects after the deadline as timeout", () => + Effect.gen(function* () { + const providerValue = "late-provider-reject-must-not-leak"; + const client = mockClient(); + createMCPClientMock.mockResolvedValue(client as never); + let now = 0; + generateTextMock.mockImplementation((() => { + now = 50; + return Promise.reject(new Error(providerValue)); + }) as never); + const testClock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + monotonicTimeNanosUnsafe: () => 0n, + monotonicTimeNanos: Effect.succeed(0n), + currentTimeNanosUnsafe: () => 0n, + currentTimeNanos: Effect.succeed(0n), + sleep: () => Effect.never, + }; + + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, timeoutMs: 10 }).pipe( + Effect.provideService(Clock.Clock, testClock), + ), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterTimeoutError); + assert.strictEqual(result.failure.timeoutMs, 10); + assert.notInclude(serialized(result.failure), providerValue); + } + assert.strictEqual(client.close.mock.calls.length, 1); + }), + ); + + it.effect("preserves parent interruption instead of succeeding or mapping a typed failure", () => + Effect.gen(function* () { + const client = mockClient(); + createMCPClientMock.mockImplementation(((config: { + initializationOptions?: { signal?: AbortSignal }; + }) => { + const { promise } = Promise.withResolvers(); + config.initializationOptions?.signal?.addEventListener("abort", () => undefined, { + once: true, + }); + return promise; + }) as never); + + const fiber = yield* Effect.forkChild(runOpenRouterPluginEvalTrial(evalCase, options)); + yield* Fiber.interrupt(fiber); + const exit = yield* Fiber.await(fiber); + + assert.strictEqual(Exit.isFailure(exit), true); + if (Exit.isFailure(exit)) { + assert.strictEqual(Cause.hasInterrupts(exit.cause), true); + } + assert.strictEqual(generateTextMock.mock.calls.length, 0); + }), + ); + + it.live("times out without waiting for a late MCP client, then closes it once", () => + Effect.gen(function* () { + const client = mockClient(); + let aborted = false; + const { promise, resolve } = Promise.withResolvers(); + createMCPClientMock.mockImplementation(((config: { + initializationOptions?: { signal?: AbortSignal }; + }) => { + config.initializationOptions?.signal?.addEventListener( + "abort", + () => { + aborted = true; + }, + { once: true }, + ); + return promise; + }) as never); + + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, timeoutMs: 20 }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterTimeoutError); + assert.notInclude(serialized(result.failure), options.apiKey); + } + assert.strictEqual(aborted, true); + assert.strictEqual(generateTextMock.mock.calls.length, 0); + assert.strictEqual(client.close.mock.calls.length, 0); + + resolve(client); + yield* Effect.sleep("30 millis"); + assert.strictEqual(client.close.mock.calls.length, 1); + }), + ); + + it.live("returns timeout without waiting for a hanging MCP close", () => + Effect.gen(function* () { + const client = mockClient({ + close: () => Promise.withResolvers().promise, + }); + createMCPClientMock.mockResolvedValue(client as never); + generateTextMock.mockImplementation(({ abortSignal }: { abortSignal?: AbortSignal }) => { + const { promise, reject } = Promise.withResolvers(); + abortSignal?.addEventListener("abort", () => reject(abortSignal.reason), { once: true }); + return promise; + }); + + const result = yield* Effect.result( + runOpenRouterPluginEvalTrial(evalCase, { ...options, timeoutMs: 20 }), + ); + + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalOpenRouterTimeoutError); + } + assert.strictEqual(client.close.mock.calls.length, 1); + }), + ); +}); diff --git a/packages/evals/__tests__/public-results.test.ts b/packages/evals/__tests__/public-results.test.ts index a443378..7e1e891 100644 --- a/packages/evals/__tests__/public-results.test.ts +++ b/packages/evals/__tests__/public-results.test.ts @@ -608,6 +608,50 @@ describe("public eval result adapter", () => { }), ); + it.effect("separates public identity by observation target at the same requested model", () => + Effect.gen(function* () { + const model = "shared-requested-model"; + const labeled = { + openrouter_api: { ...report, target: "openrouter_api", model }, + claude_cli: { ...report, target: "claude_cli", model }, + } as const; + const results = { + openrouter_api: yield* makePublicEvalResult( + options({ + reportJson: serialize(labeled.openrouter_api), + resultId: "synthetic-openrouter-api", + }), + ), + claude_cli: yield* makePublicEvalResult( + options({ + reportJson: serialize(labeled.claude_cli), + resultId: "synthetic-claude-cli", + }), + ), + }; + + assert.strictEqual(results.openrouter_api.configuration.model, model); + assert.strictEqual(results.claude_cli.configuration.model, model); + assert.strictEqual(results.openrouter_api.benchmark.target, "openrouter_api"); + assert.strictEqual(results.claude_cli.benchmark.target, "claude_cli"); + assert.notStrictEqual( + results.openrouter_api.benchmark.target, + results.claude_cli.benchmark.target, + ); + + const mismatched = yield* failureOf( + options({ + reportJson: serialize(labeled.openrouter_api), + configurationJson: configurationJson({ + model, + target: "claude_cli", + }), + }), + ); + assert.strictEqual(mismatched.reason, "configuration_mismatch"); + }), + ); + it.effect( "pins only a declaration that matches the report and leaves the default labels_only", () => @@ -643,4 +687,48 @@ describe("public eval result adapter", () => { assert.include(result.ranking.reasons, "missing_pinned_configuration"); }), ); + + it.effect("keeps slash-separated model identity through report and pinned configuration", () => + Effect.gen(function* () { + const models = ["openai/gpt-5.1", "openrouter/openai/gpt-5.1"] as const; + for (const model of models) { + const labeledJson = serialize({ ...report, model }); + const declaration = configurationJson({ model }); + const result = yield* makePublicEvalResult( + options({ + reportJson: labeledJson, + configurationJson: declaration, + }), + ); + assert.strictEqual(result.configuration.model, model); + assert.strictEqual(result.configuration.pinnedSha256, sha256(declaration)); + } + + const slashCandidate = yield* failureOf( + options({ reportJson: serialize({ ...report, candidate: "openai/gpt-5.1" }) }), + ); + assert.strictEqual(slashCandidate.reason, "invalid_report"); + + const rejectedModels = [ + "/tmp", + "../foo", + "a//b", + "https://example.com/model", + "openai/gpt-5.1?q=1", + "a".repeat(129), + ] as const; + for (const model of rejectedModels) { + const invalidReport = yield* failureOf( + options({ reportJson: serialize({ ...report, model }) }), + model, + ); + assert.strictEqual(invalidReport.reason, "invalid_report", model); + const invalidConfiguration = yield* failureOf( + options({ configurationJson: configurationJson({ model }) }), + model, + ); + assert.strictEqual(invalidConfiguration.reason, "invalid_configuration", model); + } + }), + ); }); diff --git a/packages/evals/__tests__/runner.test.ts b/packages/evals/__tests__/runner.test.ts index 78ee256..8d43dbe 100644 --- a/packages/evals/__tests__/runner.test.ts +++ b/packages/evals/__tests__/runner.test.ts @@ -1,8 +1,13 @@ import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"; import * as BunPath from "@effect/platform-bun/BunPath"; -import { isGinaReadToolName, listCatalogToolNames } from "@askgina/contracts"; +import { + catalogSha, + isGinaReadToolName, + listCatalogToolNames, + PublicEvalAttemptCaptureSchema, +} from "@askgina/contracts"; import { assert, describe, it } from "@effect/vitest"; -import { Effect, Layer, Path } from "effect"; +import { Effect, Layer, Path, Schema } from "effect"; import { decodePluginEvalObservationSet, @@ -11,13 +16,17 @@ import { loadPluginEvalObservationSet, loadPluginEvalSuite, LiveEvalSelectionError, + makePublicEvalAttemptCapture, + makePublicEvalResult, PluginEvalReplayContractError, + PluginEvalTargetSchema, + PublicEvalResultError, replayPluginEvalObservationSet, runLiveEvalSuite, runHermeticEvalReplay, } from "../src/index"; import { PublicEvalAttemptCaptureError } from "../src/public-attempts"; -import { makeSanitizedEvalRunReport } from "../src/report"; +import { makeSanitizedEvalRunReport, SanitizedEvalRunReportSchema } from "../src/report"; const collectPublicStrings = (value: unknown): readonly string[] => { if (typeof value === "string") return [value]; @@ -175,7 +184,7 @@ describe("hermetic eval replay", () => { assert.isTrue(expectedTools.every(isGinaReadToolName)); }), ); - it.effect("requires canonical catalog evidence for both completed live targets", () => + it.effect("requires canonical catalog evidence for completed live MCP targets", () => Effect.gen(function* () { const paths = yield* fixturePaths; const suite = yield* loadPluginEvalSuite(paths.liveSuite); @@ -183,7 +192,13 @@ describe("hermetic eval replay", () => { if (evalCase === undefined) return yield* Effect.die("missing fixture case"); const canonicalTools = listCatalogToolNames(); - for (const target of ["responses_api", "codex_cli"] as const) { + for (const target of [ + "responses_api", + "openrouter_api", + "codex_cli", + "claude_cli", + "omp_harness", + ] as const) { for (const availableTools of [undefined, canonicalTools.slice(1)] as const) { const result = yield* Effect.result( replayPluginEvalObservationSet(suite, { @@ -196,7 +211,7 @@ describe("hermetic eval replay", () => { catalog_version: suite.suite.catalog_version, allowed_tools: canonicalTools, candidate: "test-candidate", - target: "fixture", + target, model: "test-model", started_at: "2026-08-25T00:00:00.000Z", repetitions: 1, @@ -229,6 +244,345 @@ describe("hermetic eval replay", () => { } }), ); + it.effect("rejects typed replay identity drift before target catalog checks", () => + Effect.gen(function* () { + const paths = yield* fixturePaths; + const suite = yield* loadPluginEvalSuite(paths.liveSuite); + const evalCase = suite.cases[0]; + if (evalCase === undefined) return yield* Effect.die("missing fixture case"); + const canonicalTools = listCatalogToolNames(); + const captureModes = [undefined, { captureAttempts: true }] as const; + const manifest = { + version: 1 as const, + run_id: "typed-replay-invariants", + suite_id: suite.suite.id, + suite_version: suite.version, + catalog_version: suite.suite.catalog_version, + allowed_tools: canonicalTools, + candidate: "test-candidate", + target: "openrouter_api" as const, + model: "test-model", + started_at: "2026-08-25T00:00:00.000Z", + repetitions: 1, + clean_chat: true, + account_class: "synthetic", + artifact_policy: "sanitized" as const, + }; + const observation = { + version: 1 as const, + run_id: "typed-replay-invariants", + case_id: evalCase.id, + target: "openrouter_api" as const, + model: "test-model", + repetition: 1, + started_at: "2026-08-25T00:00:01.000Z", + status: "completed" as const, + duration_ms: 1, + tool_calls: [] as const, + available_tools: canonicalTools, + }; + const invalidSets = [ + { + needle: "displayed_model does not match the manifest", + observationSet: { + version: 1 as const, + manifest: { ...manifest, displayed_model: "Shown model" }, + observations: [observation], + }, + }, + { + needle: "run_id does not match the manifest", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, run_id: "other-run" }], + }, + }, + { + needle: "model does not match the manifest", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, model: "other-model" }], + }, + }, + { + needle: "target does not match the manifest", + observationSet: { + version: 1 as const, + manifest, + observations: [ + { + version: 1 as const, + run_id: observation.run_id, + case_id: observation.case_id, + target: "fixture" as const, + model: observation.model, + repetition: observation.repetition, + started_at: observation.started_at, + status: observation.status, + duration_ms: observation.duration_ms, + tool_calls: observation.tool_calls, + }, + ], + }, + }, + { + needle: "exceeds the manifest repetition count", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, repetition: 2 }], + }, + }, + { + needle: "repetition must be a positive safe integer", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, repetition: 1.5 }], + }, + }, + { + needle: "manifest repetitions must be a positive safe integer", + observationSet: { + version: 1 as const, + manifest: { ...manifest, repetitions: 1.5 }, + observations: [observation], + }, + }, + { + needle: "is completed but has a top-level error", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, error: "unexpected" }], + }, + }, + { + needle: "is failed but has no top-level error", + observationSet: { + version: 1 as const, + manifest, + observations: [{ ...observation, status: "failed" as const }], + }, + }, + ]; + + for (const invalid of invalidSets) { + for (const options of captureModes) { + const result = yield* Effect.result( + replayPluginEvalObservationSet(suite, invalid.observationSet, options), + ); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalReplayContractError); + if (result.failure instanceof PluginEvalReplayContractError) { + assert.include(result.failure.reasons.join("\n"), invalid.needle); + } + } + } + } + }), + ); + it.effect("does not treat duplicate repetition-1 rows as complete replay coverage", () => + Effect.gen(function* () { + const paths = yield* fixturePaths; + const suite = yield* loadPluginEvalSuite(paths.liveSuite); + const evalCase = suite.cases[0]; + if (evalCase === undefined) return yield* Effect.die("missing fixture case"); + const canonicalTools = listCatalogToolNames(); + const singleCaseSuite = { ...suite, cases: [evalCase] }; + const observationSet = { + version: 1 as const, + manifest: { + version: 1 as const, + run_id: "duplicate-repetition-coverage", + suite_id: suite.suite.id, + suite_version: suite.version, + catalog_version: suite.suite.catalog_version, + allowed_tools: canonicalTools, + candidate: "test-candidate", + target: "openrouter_api" as const, + model: "test-model", + started_at: "2026-08-25T00:00:00.000Z", + repetitions: 3, + clean_chat: true, + account_class: "synthetic", + artifact_policy: "sanitized" as const, + }, + observations: [1, 2, 3].map((index) => ({ + version: 1 as const, + run_id: "duplicate-repetition-coverage", + case_id: evalCase.id, + target: "openrouter_api" as const, + model: "test-model", + repetition: 1, + started_at: `2026-08-25T00:00:0${index}.000Z`, + status: "completed" as const, + duration_ms: index, + tool_calls: [], + available_tools: canonicalTools, + })), + }; + + for (const options of [undefined, { captureAttempts: true }] as const) { + const result = yield* Effect.result( + replayPluginEvalObservationSet(singleCaseSuite, observationSet, options), + ); + assert.strictEqual(result._tag, "Failure"); + if (result._tag === "Failure") { + assert.instanceOf(result.failure, PluginEvalReplayContractError); + if (result.failure instanceof PluginEvalReplayContractError) { + assert.include( + result.failure.reasons.join("\n"), + `Duplicate observation attempt: ${evalCase.id}#1`, + ); + } + } + } + }), + ); + it.effect( + "keeps new live targets distinct without inventing displayed-model or skill evidence", + () => + Effect.gen(function* () { + const paths = yield* fixturePaths; + const suite = yield* loadPluginEvalSuite(paths.liveSuite); + const evalCase = suite.cases[0]; + if (evalCase === undefined) return yield* Effect.die("missing fixture case"); + const expectedProvenance = { + suiteId: suite.suite.id, + suiteVersion: suite.version, + fixtureVersion: 1, + catalogSha, + } as const; + const artifacts: Partial< + Record< + "openrouter_api" | "claude_cli" | "omp_harness", + { + readonly reportJson: string; + readonly attemptCaptureJson: string; + readonly reportSha256: string; + } + > + > = {}; + + for (const target of ["openrouter_api", "claude_cli", "omp_harness"] as const) { + assert.isFalse(Schema.is(PluginEvalTargetSchema)(`${target}_alias`)); + + const { report, attempts } = yield* runLiveEvalSuite( + { + suite, + caseIds: [evalCase.id], + runId: "shared-target-run", + candidate: "test-candidate", + target, + model: "provider/requested-model", + reasoning: "test", + repetitions: 3, + accountClass: "synthetic", + captureAttempts: true, + }, + (input) => + Effect.succeed({ + version: 1, + run_id: input.runId, + case_id: input.evalCase.id, + target: input.target, + model: input.model, + repetition: input.repetition, + started_at: input.startedAt, + status: "completed" as const, + duration_ms: input.repetition, + tool_calls: [ + { + sequence: 0, + name: "gina.listScheduledPrompts", + arguments: {}, + result_bytes: 0, + requested_scope: "tools:read", + }, + ], + available_tools: listCatalogToolNames(), + }), + ); + + assert.strictEqual(report.target, target); + assert.strictEqual(report.model, "provider/requested-model"); + assert.strictEqual(report.aggregate.skillActivation.passed, 0); + assert.strictEqual(report.aggregate.skillActivation.failed, 0); + if (attempts === null) { + return yield* Effect.die("opt-in capture returned no summaries"); + } + assert.strictEqual(attempts.length, 3); + assert.isTrue( + attempts.every((attempt) => attempt.checks.skillActivation === "not_applicable"), + ); + + const encodedReport = yield* Schema.encodeEffect( + Schema.fromJsonString(SanitizedEvalRunReportSchema, { space: 2 }), + )(report); + const reportJson = `${encodedReport}\n`; + const capture = yield* makePublicEvalAttemptCapture({ + runId: report.runId, + reportContent: reportJson, + attempts, + }); + const encodedCapture = yield* Schema.encodeEffect( + Schema.fromJsonString(PublicEvalAttemptCaptureSchema, { space: 2 }), + )(capture); + const attemptCaptureJson = `${encodedCapture}\n`; + const publicResult = yield* makePublicEvalResult({ + reportJson, + attemptCaptureJson, + resultId: `result-${target}`, + dataOrigin: "synthetic", + expectedProvenance, + }); + assert.strictEqual(publicResult.benchmark.target, target); + assert.strictEqual(publicResult.configuration.model, "provider/requested-model"); + assert.strictEqual(publicResult.source.kind, "sanitized_aggregate_with_attempts"); + assert.strictEqual(publicResult.evidence.attemptDetail, "available"); + artifacts[target] = { + reportJson, + attemptCaptureJson, + reportSha256: publicResult.source.reportSha256, + }; + } + + const openRouter = artifacts.openrouter_api; + const claude = artifacts.claude_cli; + const omp = artifacts.omp_harness; + if (openRouter === undefined || claude === undefined || omp === undefined) { + return yield* Effect.die("target artifacts were not captured"); + } + assert.strictEqual( + new Set([openRouter.reportSha256, claude.reportSha256, omp.reportSha256]).size, + 3, + ); + + const crossTargetCapture = yield* Effect.result( + makePublicEvalResult({ + reportJson: openRouter.reportJson, + attemptCaptureJson: omp.attemptCaptureJson, + resultId: "cross-target-capture", + dataOrigin: "synthetic", + expectedProvenance, + }), + ); + assert.strictEqual(crossTargetCapture._tag, "Failure"); + if (crossTargetCapture._tag === "Failure") { + assert.instanceOf(crossTargetCapture.failure, PublicEvalResultError); + if (crossTargetCapture.failure instanceof PublicEvalResultError) { + assert.strictEqual( + crossTargetCapture.failure.reason, + "attempt_capture_hash_mismatch", + ); + } + } + }), + ); it.effect("rejects out-of-catalog expectation tools before invoking a transport", () => Effect.gen(function* () { const paths = yield* fixturePaths; diff --git a/packages/evals/package.json b/packages/evals/package.json index 9fec4db..a4a430a 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -21,12 +21,24 @@ "check:marketplace:codex": "bun run build && bun dist/bin/check-codex-marketplace.js" }, "dependencies": { + "@ai-sdk/harness": "1.0.102", + "@ai-sdk/harness-acp": "1.0.40", + "@ai-sdk/mcp": "2.0.45", "@askgina/contracts": "workspace:*", "@askgina/plugin-core": "workspace:*", "@askgina/sdk": "workspace:*", "@effect/platform-bun": "4.0.0-rc.111", + "@openrouter/ai-sdk-provider": "3.0.0", + "ai": "7.0.93", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "dockerode": "4.0.12", "effect": "4.0.0-rc.111", - "yaml": "2.8.3" + "yaml": "2.8.3", + "zod": "4.1.8" + }, + "devDependencies": { + "@types/dockerode": "4.0.1" }, "engines": { "bun": "1.4.x" diff --git a/packages/evals/src/bin/live.ts b/packages/evals/src/bin/live.ts index f3ae755..92ff79e 100755 --- a/packages/evals/src/bin/live.ts +++ b/packages/evals/src/bin/live.ts @@ -27,6 +27,7 @@ import { Stream, } from "effect"; +import { runClaudeCliPluginEvalTrial } from "../claude-cli"; import { CODEX_CLI_ALLOWED_ENVIRONMENT_NAMES, attestCodexExecutable, @@ -42,6 +43,13 @@ import { preflightLiveEvalSuite, runLiveEvalSuite, } from "../live"; +import { + isOmpProvider, + prepareOmpHarnessRuntime, + runOmpHarnessPluginEvalTrial, + type OmpProvider, +} from "../omp-harness"; +import { runOpenRouterPluginEvalTrial } from "../openrouter"; import { assertPublicEvalAttemptOutputPath, assertPublicEvalAttemptPlan, @@ -62,7 +70,53 @@ const decodeJsonObjectOption = Schema.decodeUnknownOption(JsonObjectString); const decodeUnknownJsonOption = Schema.decodeUnknownOption(UnknownJsonString); const encodeUnknownJson = Schema.encodeEffect(UnknownJsonString); const encodePrettyUnknownJson = Schema.encodeEffect(PrettyUnknownJsonString); -type LiveEvalRunner = "codex" | "responses"; +export const DEFAULT_OPENROUTER_MAX_STEPS = 8; +export const DEFAULT_CLAUDE_MAX_TURNS = 8; +export const MAXIMUM_LIVE_EVAL_TOOL_BUDGET = 32; +const CLAUDE_PLUGIN_DIRECTORY_SEGMENTS = ["plugins", "ask-gina", "targets", "claude"] as const; +const CLAUDE_SKILL_DIRECTORY_SEGMENTS = ["plugins", "ask-gina", "skills"] as const; +const ASK_GINA_ACCESS_TOKEN = "ASK_GINA_ACCESS_TOKEN"; +const OPENAI_API_KEY = "OPENAI_API_KEY"; +const OPENROUTER_API_KEY = "OPENROUTER_API_KEY"; +const ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"; +const CLAUDE_EVAL_EXECUTABLE = "CLAUDE_EVAL_EXECUTABLE"; +const CODEX_EVAL_EXECUTABLE = "CODEX_EVAL_EXECUTABLE"; +const CODEX_EVAL_EXECUTABLE_SHA256 = "CODEX_EVAL_EXECUTABLE_SHA256"; +const OMP_EVAL_API_KEY = "OMP_EVAL_API_KEY"; +const OMP_EVAL_EXECUTABLE = "OMP_EVAL_EXECUTABLE"; +const OMP_EVAL_EXECUTABLE_SHA256 = "OMP_EVAL_EXECUTABLE_SHA256"; + +export type LiveEvalRunner = "codex" | "responses" | "openrouter" | "claude" | "omp"; + +const LIVE_EVAL_TARGET = { + responses: "responses_api", + codex: "codex_cli", + openrouter: "openrouter_api", + claude: "claude_cli", + omp: "omp_harness", +} as const; + +export interface LiveEvalTrialDispatch { + readonly target: (typeof LIVE_EVAL_TARGET)[LiveEvalRunner]; + readonly displayedModel?: string; + readonly maxSteps?: number; + readonly maxTurns?: number; + readonly model?: string; +} + +export const liveEvalTrialDispatch = (options: LiveEvalCliOptions): LiveEvalTrialDispatch => { + const target = LIVE_EVAL_TARGET[options.runner]; + if (options.runner === "openrouter") { + return { target, maxSteps: options.maxSteps }; + } + if (options.runner === "claude") { + return { target, maxTurns: options.maxTurns }; + } + if (options.runner === "omp") { + return { target, model: `${options.provider}/${options.model}` }; + } + return { target, displayedModel: options.model }; +}; interface CodexEvalRuntime { readonly executable: AttestedCodexExecutable; @@ -72,8 +126,13 @@ interface CodexEvalRuntime { readonly pluginSkillRoot: string; } -interface LiveEvalCliOptions { - readonly runner: LiveEvalRunner; +interface ClaudeEvalRuntime { + readonly executablePath: string; + readonly workingDirectory: string; + readonly pluginDirectory: string; +} + +interface LiveEvalCliSharedOptions { readonly suitePath: string; readonly runId: string; readonly candidate: string; @@ -86,21 +145,72 @@ interface LiveEvalCliOptions { readonly attemptsOutputPath?: string; } -class LiveEvalCliError extends Data.TaggedError("LiveEvalCliError")<{ +export type LiveEvalCliOptions = + | (LiveEvalCliSharedOptions & { readonly runner: "responses" | "codex" }) + | (LiveEvalCliSharedOptions & { readonly runner: "openrouter"; readonly maxSteps: number }) + | (LiveEvalCliSharedOptions & { readonly runner: "claude"; readonly maxTurns: number }) + | (LiveEvalCliSharedOptions & { readonly runner: "omp"; readonly provider: OmpProvider }); + +export type LiveEvalCliParseResult = + | { readonly mode: "help"; readonly usage: string } + | { readonly mode: "run"; readonly options: LiveEvalCliOptions }; + +export class LiveEvalCliError extends Data.TaggedError("LiveEvalCliError")<{ readonly reason: | "dirty-source" | "git-preflight-failed" | "invalid-arguments" | "invalid-credentials" | "codex-preflight-failed" + | "claude-preflight-failed" + | "omp-preflight-failed" | "trial-failed" | "catalog-preflight-failed" | "report-exists" | "report-write-failed"; + readonly missing?: readonly string[]; }> {} -const usage = - "Usage: bun run eval: -- --suite --run-id --candidate --model --reasoning --repetitions <3..5> --account-class [--case ] --timeout-ms [--attempts-output ]"; +const REQUIRED_LIVE_EVAL_FLAGS = + "--suite --run-id --candidate --model --reasoning --repetitions <3..5> --account-class [--case ] --timeout-ms [--attempts-output ]"; + +export const formatLiveEvalCliUsage = (runner?: LiveEvalRunner): string => { + if (runner === "openrouter") { + return [ + `Usage: bun run eval:openrouter -- ${REQUIRED_LIVE_EVAL_FLAGS} [--max-steps <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>]`, + `--max-steps default ${DEFAULT_OPENROUTER_MAX_STEPS}. Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OPENROUTER_API_KEY}`, + ].join("\n"); + } + if (runner === "claude") { + return [ + `Usage: bun run eval:claude -- ${REQUIRED_LIVE_EVAL_FLAGS} [--max-turns <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>]`, + `--max-turns default ${DEFAULT_CLAUDE_MAX_TURNS}. Environment: ${ASK_GINA_ACCESS_TOKEN}, ${ANTHROPIC_API_KEY}, ${CLAUDE_EVAL_EXECUTABLE}`, + ].join("\n"); + } + if (runner === "codex") { + return [ + `Usage: bun run eval:codex -- ${REQUIRED_LIVE_EVAL_FLAGS}`, + `Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OPENAI_API_KEY}, ${CODEX_EVAL_EXECUTABLE}, ${CODEX_EVAL_EXECUTABLE_SHA256}`, + ].join("\n"); + } + if (runner === "responses") { + return [ + `Usage: bun run eval:responses -- ${REQUIRED_LIVE_EVAL_FLAGS}`, + `Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OPENAI_API_KEY}`, + ].join("\n"); + } + if (runner === "omp") { + return [ + `Usage: bun run eval:omp -- ${REQUIRED_LIVE_EVAL_FLAGS} --provider `, + `Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OMP_EVAL_API_KEY}, ${OMP_EVAL_EXECUTABLE}, ${OMP_EVAL_EXECUTABLE_SHA256}`, + ].join("\n"); + } + return [ + `Usage: bun run eval: -- ${REQUIRED_LIVE_EVAL_FLAGS}`, + `OpenRouter-only: [--max-steps <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_OPENROUTER_MAX_STEPS}). Claude-only: [--max-turns <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_CLAUDE_MAX_TURNS}). OMP-only: --provider .`, + `Environment: ${ASK_GINA_ACCESS_TOKEN} always; ${OPENAI_API_KEY} (responses, codex); ${OPENROUTER_API_KEY} (openrouter); ${ANTHROPIC_API_KEY} and ${CLAUDE_EVAL_EXECUTABLE} (claude); ${CODEX_EVAL_EXECUTABLE} and ${CODEX_EVAL_EXECUTABLE_SHA256} (codex); ${OMP_EVAL_API_KEY}, ${OMP_EVAL_EXECUTABLE}, and ${OMP_EVAL_EXECUTABLE_SHA256} (omp).`, + ].join("\n"); +}; const parsePositiveInteger = (value: string | undefined): number | undefined => { if (value === undefined || !/^\d+$/u.test(value)) return undefined; @@ -108,9 +218,23 @@ const parsePositiveInteger = (value: string | undefined): number | undefined => return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined; }; -const parseOptions = ( +const parseToolBudget = (value: string | undefined): number | undefined => { + const parsed = parsePositiveInteger(value); + return parsed !== undefined && parsed <= MAXIMUM_LIVE_EVAL_TOOL_BUDGET ? parsed : undefined; +}; + +const parseLiveEvalRunner = (value: string | undefined): LiveEvalRunner | undefined => + value === "responses" || + value === "codex" || + value === "openrouter" || + value === "claude" || + value === "omp" + ? value + : undefined; + +export const parseLiveEvalCliOptions = ( argv: readonly string[], -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { let runner: LiveEvalRunner | undefined; let suitePath: string | undefined; @@ -122,8 +246,17 @@ const parseOptions = ( let accountClass: string | undefined; let timeoutMs: number | undefined; let attemptsOutputPath: string | undefined; + let maxSteps: number | undefined; + let maxTurns: number | undefined; + let provider: OmpProvider | undefined; const caseIds: string[] = []; const seenFlags = new Set(); + const help = argv.some((flag) => flag === "--help" || flag === "-h"); + if (help) { + const runnerFlag = argv.findIndex((flag) => flag === "--runner"); + const helpRunner = runnerFlag === -1 ? undefined : parseLiveEvalRunner(argv[runnerFlag + 1]); + return { mode: "help", usage: formatLiveEvalCliUsage(helpRunner) }; + } for (let index = 0; index < argv.length; index += 1) { const flag = argv[index]; @@ -139,12 +272,14 @@ const parseOptions = ( } seenFlags.add(flag); switch (flag) { - case "--runner": - if (value !== "responses" && value !== "codex") { + case "--runner": { + const parsedRunner = parseLiveEvalRunner(value); + if (parsedRunner === undefined) { return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); } - runner = value; + runner = parsedRunner; break; + } case "--suite": suitePath = value; break; @@ -175,6 +310,24 @@ const parseOptions = ( case "--attempts-output": attemptsOutputPath = value; break; + case "--max-steps": + maxSteps = parseToolBudget(value); + if (maxSteps === undefined) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + break; + case "--max-turns": + maxTurns = parseToolBudget(value); + if (maxTurns === undefined) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + break; + case "--provider": + if (isOmpProvider(value)) { + provider = value; + break; + } + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); default: return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); } @@ -197,8 +350,17 @@ const parseOptions = ( return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); } - return { - runner, + if (runner !== "openrouter" && seenFlags.has("--max-steps")) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + if (runner !== "claude" && seenFlags.has("--max-turns")) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + if (runner !== "omp" && seenFlags.has("--provider")) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + + const shared = { suitePath, runId, candidate, @@ -209,7 +371,30 @@ const parseOptions = ( ...(caseIds.length === 0 ? {} : { caseIds }), timeoutMs, ...(attemptsOutputPath === undefined ? {} : { attemptsOutputPath }), - }; + } satisfies LiveEvalCliSharedOptions; + + if (runner === "openrouter") { + return { + mode: "run", + options: { ...shared, runner, maxSteps: maxSteps ?? DEFAULT_OPENROUTER_MAX_STEPS }, + }; + } + if (runner === "claude") { + return { + mode: "run", + options: { ...shared, runner, maxTurns: maxTurns ?? DEFAULT_CLAUDE_MAX_TURNS }, + }; + } + if (runner === "omp") { + if (provider === undefined) { + return yield* new LiveEvalCliError({ reason: "invalid-arguments" }); + } + return { + mode: "run", + options: { ...shared, runner, provider }, + }; + } + return { mode: "run", options: { ...shared, runner } }; }); const loadCodexEnvironment = () => @@ -260,14 +445,105 @@ const requireCleanSource = ( }), ); +const missingCredentials = (missing: readonly string[]) => + new LiveEvalCliError({ reason: "invalid-credentials", missing }); + const requireRedacted = ( + name: string, value: Redacted.Redacted, -): Effect.Effect => { +): Effect.Effect, LiveEvalCliError> => { const trimmed = Redacted.value(value).trim(); return trimmed.length === 0 - ? Effect.fail(new LiveEvalCliError({ reason: "invalid-credentials" })) + ? Effect.fail(missingCredentials([name])) : Effect.succeed(Redacted.make(trimmed)); }; + +const loadRedactedEnv = (name: string) => + Config.redacted(name).pipe( + Effect.mapError(() => missingCredentials([name])), + Effect.flatMap((value) => requireRedacted(name, value)), + ); + +const loadNonEmptyEnv = (name: string) => + Config.string(name).pipe( + Effect.map((value) => value.trim()), + Effect.mapError(() => missingCredentials([name])), + Effect.filterOrFail( + (value) => value.length > 0, + () => missingCredentials([name]), + ), + ); + +export type LiveEvalCredentials = + | { + readonly runner: "responses"; + readonly accessToken: Redacted.Redacted; + readonly openAiApiKey: Redacted.Redacted; + } + | { + readonly runner: "codex"; + readonly accessToken: Redacted.Redacted; + readonly openAiApiKey: Redacted.Redacted; + readonly executablePath: string; + readonly expectedSha256: string; + } + | { + readonly runner: "openrouter"; + readonly accessToken: Redacted.Redacted; + readonly openRouterApiKey: Redacted.Redacted; + } + | { + readonly runner: "claude"; + readonly accessToken: Redacted.Redacted; + readonly apiKey: Redacted.Redacted; + readonly executablePath: string; + } + | { + readonly runner: "omp"; + readonly accessToken: Redacted.Redacted; + readonly apiKey: Redacted.Redacted; + readonly executablePath: string; + readonly expectedSha256: string; + }; + +export const loadLiveEvalCredentials = ( + runner: LiveEvalRunner, +): Effect.Effect => + Effect.gen(function* () { + const accessToken = yield* loadRedactedEnv(ASK_GINA_ACCESS_TOKEN); + if (runner === "responses") { + const openAiApiKey = yield* loadRedactedEnv(OPENAI_API_KEY); + return { runner, accessToken, openAiApiKey }; + } + if (runner === "openrouter") { + const openRouterApiKey = yield* loadRedactedEnv(OPENROUTER_API_KEY); + return { runner, accessToken, openRouterApiKey }; + } + if (runner === "claude") { + const path = yield* Path.Path; + const apiKey = yield* loadRedactedEnv(ANTHROPIC_API_KEY); + const executablePath = yield* loadNonEmptyEnv(CLAUDE_EVAL_EXECUTABLE); + if (!path.isAbsolute(executablePath)) { + return yield* missingCredentials([CLAUDE_EVAL_EXECUTABLE]); + } + return { runner, accessToken, apiKey, executablePath }; + } + if (runner === "omp") { + const path = yield* Path.Path; + const apiKey = yield* loadRedactedEnv(OMP_EVAL_API_KEY); + const executablePath = yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE); + const expectedSha256 = (yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE_SHA256)).toLowerCase(); + if (!path.isAbsolute(executablePath)) { + return yield* missingCredentials([OMP_EVAL_EXECUTABLE]); + } + return { runner, accessToken, apiKey, executablePath, expectedSha256 }; + } + const openAiApiKey = yield* loadRedactedEnv(OPENAI_API_KEY); + const executablePath = yield* loadNonEmptyEnv(CODEX_EVAL_EXECUTABLE); + const expectedSha256 = (yield* loadNonEmptyEnv(CODEX_EVAL_EXECUTABLE_SHA256)).toLowerCase(); + return { runner, accessToken, openAiApiKey, executablePath, expectedSha256 }; + }); + const isJsonObject = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); @@ -491,6 +767,91 @@ const seedCodexOAuthCredential = ( return credentialPath; }); +const setupClaudeRuntime = (root: string, executablePath: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const fail = () => new LiveEvalCliError({ reason: "claude-preflight-failed" }); + const requireContained = (candidate: string) => { + const relative = path.relative(root, candidate); + return relative === "" || relative.startsWith("..") || path.isAbsolute(relative) + ? Effect.fail(fail()) + : Effect.void; + }; + const isSymbolicLink = (target: string) => + fs.readLink(target).pipe(Effect.match({ onFailure: () => false, onSuccess: () => true })); + const copyCheckedFile = ( + source: string, + destination: string, + ): Effect.Effect => + Effect.gen(function* () { + if (yield* isSymbolicLink(source)) return yield* fail(); + const info = yield* fs.stat(source).pipe(Effect.mapError(() => fail())); + if (info.type !== "File") return yield* fail(); + const bytes = yield* fs.readFile(source).pipe(Effect.mapError(() => fail())); + if (bytes.byteLength !== Number(info.size) || bytes.byteLength === 0) return yield* fail(); + yield* fs + .makeDirectory(path.dirname(destination), { recursive: true }) + .pipe(Effect.mapError(() => fail())); + yield* fs.writeFile(destination, bytes, { flag: "wx" }).pipe(Effect.mapError(() => fail())); + }); + const copyCheckedTree = ( + source: string, + destination: string, + ): Effect.Effect => + Effect.gen(function* () { + if (yield* isSymbolicLink(source)) return yield* fail(); + const info = yield* fs.stat(source).pipe(Effect.mapError(() => fail())); + if (info.type === "File") return yield* copyCheckedFile(source, destination); + if (info.type !== "Directory") return yield* fail(); + yield* fs + .makeDirectory(destination, { recursive: true }) + .pipe(Effect.mapError(() => fail())); + const entries = yield* fs.readDirectory(source).pipe(Effect.mapError(() => fail())); + yield* Effect.forEach( + entries, + (entry) => copyCheckedTree(path.join(source, entry), path.join(destination, entry)), + { discard: true }, + ); + }); + + const temporaryRoot = yield* fs + .makeTempDirectoryScoped({ prefix: "ask-gina-claude-eval-" }) + .pipe(Effect.mapError(() => fail())); + const workingDirectory = path.join(temporaryRoot, "work"); + const stagedPlugin = path.join(temporaryRoot, "plugin"); + yield* fs + .makeDirectory(workingDirectory, { recursive: true }) + .pipe(Effect.mapError(() => fail())); + const overlaySource = yield* fs + .realPath(path.join(root, ...CLAUDE_PLUGIN_DIRECTORY_SEGMENTS)) + .pipe(Effect.mapError(() => fail())); + const skillsSource = yield* fs + .realPath(path.join(root, ...CLAUDE_SKILL_DIRECTORY_SEGMENTS)) + .pipe(Effect.mapError(() => fail())); + yield* requireContained(overlaySource); + yield* requireContained(skillsSource); + yield* copyCheckedTree(overlaySource, stagedPlugin); + yield* Effect.forEach( + ASK_GINA_SKILL_DEFINITIONS, + (skill) => + Effect.gen(function* () { + const destination = path.join(stagedPlugin, "skills", skill.name); + yield* copyCheckedTree(path.join(skillsSource, skill.name), destination); + yield* fs + .remove(path.join(destination, "agents"), { recursive: true, force: true }) + .pipe(Effect.mapError(() => fail())); + }), + { discard: true }, + ); + const pluginDirectory = yield* fs.realPath(stagedPlugin).pipe(Effect.mapError(() => fail())); + return { + executablePath, + workingDirectory, + pluginDirectory, + } satisfies ClaudeEvalRuntime; + }); + const requireLiveCatalog = (accessToken: Redacted.Redacted) => { const client = createClient({ accessToken: Redacted.value(accessToken), @@ -539,6 +900,23 @@ const errorTag = (error: unknown): string => { return typeof tag === "string" ? tag : "LiveEvalTrialError"; }; +export const formatLiveEvalCliFailure = (error: unknown): string => { + if (errorTag(error) === "LiveEvalCliError" && typeof error === "object" && error !== null) { + if (Reflect.get(error, "reason") === "invalid-arguments") { + return formatLiveEvalCliUsage(); + } + const missing = Reflect.get(error, "missing"); + if ( + Array.isArray(missing) && + missing.length > 0 && + missing.every((name) => typeof name === "string") + ) { + return `live eval failed (LiveEvalCliError): missing ${missing.join(", ")}`; + } + } + return `live eval failed (${errorTag(error)})`; +}; + const writeReport = (outputPath: string, content: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -553,154 +931,223 @@ const writeReport = (outputPath: string, content: string) => }); const run = (options: LiveEvalCliOptions) => - Effect.gen(function* () { - const root = process.cwd(); - const path = yield* Path.Path; - const target = options.runner === "responses" ? "responses_api" : "codex_cli"; - const outputPath = path.join( - root, - ".plugin-eval-runs", - `${target}-${options.candidate}-${options.runId}.json`, - ); - if (options.attemptsOutputPath !== undefined) { - yield* assertPublicEvalAttemptOutputPath(options.attemptsOutputPath, outputPath); - } - const suite = yield* loadPluginEvalSuite(options.suitePath); - yield* preflightLiveEvalSuite(suite).pipe( - Effect.mapError(() => new LiveEvalCliError({ reason: "catalog-preflight-failed" })), - ); - if (options.attemptsOutputPath !== undefined) { - yield* assertPublicEvalAttemptPlan( - options.runId, - options.caseIds ?? suite.cases.map((evalCase) => evalCase.id), - options.repetitions, - ); - } - const codexEnvironment = yield* loadCodexEnvironment(); - yield* requireCleanSource(root, codexEnvironment); - let codexRuntime: CodexEvalRuntime | undefined; - if (options.runner === "codex") { - const executablePath = yield* Config.string("CODEX_EVAL_EXECUTABLE").pipe( - Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" })), + Effect.scoped( + Effect.gen(function* () { + const root = process.cwd(); + const path = yield* Path.Path; + const credentials = yield* loadLiveEvalCredentials(options.runner); + const dispatch = liveEvalTrialDispatch(options); + const model = dispatch.model ?? options.model; + const outputPath = path.join( + root, + ".plugin-eval-runs", + `${dispatch.target}-${options.candidate}-${options.runId}.json`, ); - const expectedSha256 = yield* Config.string("CODEX_EVAL_EXECUTABLE_SHA256").pipe( - Effect.map((value) => value.trim().toLowerCase()), - Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" })), + if (options.attemptsOutputPath !== undefined) { + yield* assertPublicEvalAttemptOutputPath(options.attemptsOutputPath, outputPath); + } + const suite = yield* loadPluginEvalSuite(options.suitePath); + yield* preflightLiveEvalSuite(suite).pipe( + Effect.mapError(() => new LiveEvalCliError({ reason: "catalog-preflight-failed" })), ); - const executable = yield* attestCodexExecutable({ - executablePath, - expectedSha256, - forbiddenRoots: [root], - }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" }))); - codexRuntime = yield* setupCodexRuntime(root, executable, codexEnvironment); - } - const accessToken = yield* Config.redacted("ASK_GINA_ACCESS_TOKEN").pipe( - Effect.flatMap(requireRedacted), - Effect.mapError(() => new LiveEvalCliError({ reason: "invalid-credentials" })), - ); - const apiKey = yield* Config.redacted("OPENAI_API_KEY").pipe( - Effect.flatMap(requireRedacted), - Effect.mapError(() => new LiveEvalCliError({ reason: "invalid-credentials" })), - ); - const availableTools = yield* requireLiveCatalog(accessToken); - if (codexRuntime !== undefined) { - yield* seedCodexOAuthCredential(codexRuntime, accessToken); - yield* requireCodexPluginAuth(codexRuntime, codexEnvironment); - } - - const { report, attempts } = yield* runLiveEvalSuite< - LiveEvalCliError, - HttpClient.HttpClient | ChildProcessSpawner | FileSystem.FileSystem | Path.Path - >( - { - suite, - ...(options.caseIds === undefined ? {} : { caseIds: options.caseIds }), - runId: options.runId, - candidate: options.candidate, - target, - model: options.model, - displayedModel: options.model, - reasoning: options.reasoning, - repetitions: options.repetitions, - accountClass: options.accountClass, - captureAttempts: options.attemptsOutputPath !== undefined, - }, - (input) => { - if (options.runner === "responses") { - return runResponsesApiPluginEvalTrial(input.evalCase, { - apiKey: Redacted.value(apiKey), - mcpAuthorization: Redacted.value(accessToken), - model: options.model, - reasoning: options.reasoning, - runId: input.runId, - repetition: input.repetition, - serverUrl: PRODUCTION_MCP_URL, - allowedTools: listCatalogToolNames(), - timeoutMs: options.timeoutMs, - }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); + if (options.attemptsOutputPath !== undefined) { + yield* assertPublicEvalAttemptPlan( + options.runId, + options.caseIds ?? suite.cases.map((evalCase) => evalCase.id), + options.repetitions, + ); + } + const isolatedEnvironment = yield* loadCodexEnvironment(); + yield* requireCleanSource(root, isolatedEnvironment); + let codexRuntime: CodexEvalRuntime | undefined; + let claudeRuntime: ClaudeEvalRuntime | undefined; + let ompRuntimeDirectory: string | undefined; + if (options.runner === "codex") { + if (credentials.runner !== "codex") { + return yield* new LiveEvalCliError({ reason: "codex-preflight-failed" }); } - if (codexRuntime === undefined) { - return Effect.fail(new LiveEvalCliError({ reason: "codex-preflight-failed" })); + const executable = yield* attestCodexExecutable({ + executablePath: credentials.executablePath, + expectedSha256: credentials.expectedSha256, + forbiddenRoots: [root], + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "codex-preflight-failed" }))); + codexRuntime = yield* setupCodexRuntime(root, executable, isolatedEnvironment); + } + if (options.runner === "claude") { + if (credentials.runner !== "claude") { + return yield* new LiveEvalCliError({ reason: "claude-preflight-failed" }); + } + claudeRuntime = yield* setupClaudeRuntime(root, credentials.executablePath); + } + if (options.runner === "omp") { + if (credentials.runner !== "omp") { + return yield* new LiveEvalCliError({ reason: "omp-preflight-failed" }); } - return runCodexCliPluginEvalTrial(input.evalCase, { - openAiApiKey: apiKey, - model: options.model, - displayedModel: options.model, + const prepared = yield* prepareOmpHarnessRuntime({ + root, + executablePath: credentials.executablePath, + expectedSha256: credentials.expectedSha256, + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "omp-preflight-failed" }))); + ompRuntimeDirectory = prepared.runtimeDirectory; + } + const availableTools = yield* requireLiveCatalog(credentials.accessToken); + if (codexRuntime !== undefined) { + yield* seedCodexOAuthCredential(codexRuntime, credentials.accessToken); + yield* requireCodexPluginAuth(codexRuntime, isolatedEnvironment); + } + + const { report, attempts } = yield* runLiveEvalSuite< + LiveEvalCliError, + HttpClient.HttpClient | ChildProcessSpawner | FileSystem.FileSystem | Path.Path + >( + { + suite, + ...(options.caseIds === undefined ? {} : { caseIds: options.caseIds }), + runId: options.runId, + candidate: options.candidate, + target: dispatch.target, + model, + ...(dispatch.displayedModel === undefined + ? {} + : { displayedModel: dispatch.displayedModel }), reasoning: options.reasoning, - runId: input.runId, - repetition: input.repetition, - workingDirectory: codexRuntime.workingDirectory, - executable: codexRuntime.executable, - codexHome: codexRuntime.codexHome, - pluginId: codexRuntime.pluginId, - pluginSkillRoot: codexRuntime.pluginSkillRoot, - parentEnvironment: codexEnvironment, - availableTools, - timeoutMs: options.timeoutMs, - }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); - }, - ); - const encoded = yield* encodePrettyUnknownJson(report).pipe( - Effect.map((json) => `${json}\n`), - Effect.mapError(() => new LiveEvalCliError({ reason: "report-write-failed" })), - ); - let capture: PublicEvalAttemptCapture | undefined; - if (options.attemptsOutputPath !== undefined) { - if (attempts === null) { - return yield* new LiveEvalCliError({ reason: "report-write-failed" }); + repetitions: options.repetitions, + accountClass: options.accountClass, + captureAttempts: options.attemptsOutputPath !== undefined, + }, + (input) => { + switch (options.runner) { + case "responses": + if (credentials.runner !== "responses") { + return Effect.fail(missingCredentials([OPENAI_API_KEY])); + } + return runResponsesApiPluginEvalTrial(input.evalCase, { + apiKey: Redacted.value(credentials.openAiApiKey), + mcpAuthorization: Redacted.value(credentials.accessToken), + model: options.model, + reasoning: options.reasoning, + runId: input.runId, + repetition: input.repetition, + serverUrl: PRODUCTION_MCP_URL, + allowedTools: listCatalogToolNames(), + timeoutMs: options.timeoutMs, + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); + case "openrouter": + if (credentials.runner !== "openrouter") { + return Effect.fail(missingCredentials([OPENROUTER_API_KEY])); + } + return runOpenRouterPluginEvalTrial(input.evalCase, { + apiKey: Redacted.value(credentials.openRouterApiKey), + mcpAuthorization: Redacted.value(credentials.accessToken), + model: options.model, + reasoning: options.reasoning, + runId: input.runId, + repetition: input.repetition, + serverUrl: PRODUCTION_MCP_URL, + allowedTools: listCatalogToolNames(), + timeoutMs: options.timeoutMs, + maxSteps: dispatch.maxSteps, + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); + case "claude": + if (credentials.runner !== "claude" || claudeRuntime === undefined) { + return Effect.fail(new LiveEvalCliError({ reason: "claude-preflight-failed" })); + } + return runClaudeCliPluginEvalTrial(input.evalCase, { + runId: input.runId, + repetition: input.repetition, + availableTools, + workingDirectory: claudeRuntime.workingDirectory, + executablePath: claudeRuntime.executablePath, + pluginDirectory: claudeRuntime.pluginDirectory, + mcpAuthorization: credentials.accessToken, + apiKey: credentials.apiKey, + model: options.model, + reasoning: options.reasoning, + parentEnvironment: isolatedEnvironment, + timeoutMs: options.timeoutMs, + maxTurns: dispatch.maxTurns, + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); + case "codex": + if (credentials.runner !== "codex" || codexRuntime === undefined) { + return Effect.fail(new LiveEvalCliError({ reason: "codex-preflight-failed" })); + } + return runCodexCliPluginEvalTrial(input.evalCase, { + openAiApiKey: credentials.openAiApiKey, + model: options.model, + displayedModel: options.model, + reasoning: options.reasoning, + runId: input.runId, + repetition: input.repetition, + workingDirectory: codexRuntime.workingDirectory, + executable: codexRuntime.executable, + codexHome: codexRuntime.codexHome, + pluginId: codexRuntime.pluginId, + pluginSkillRoot: codexRuntime.pluginSkillRoot, + parentEnvironment: isolatedEnvironment, + availableTools, + timeoutMs: options.timeoutMs, + }).pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" }))); + case "omp": + if (credentials.runner !== "omp" || ompRuntimeDirectory === undefined) { + return Effect.fail(new LiveEvalCliError({ reason: "omp-preflight-failed" })); + } + return runOmpHarnessPluginEvalTrial(input.evalCase, { + runId: input.runId, + repetition: input.repetition, + availableTools, + runtimeDirectory: ompRuntimeDirectory, + provider: options.provider, + model: options.model, + reasoning: options.reasoning, + apiKey: credentials.apiKey, + mcpAuthorization: credentials.accessToken, + timeoutMs: options.timeoutMs, + }).pipe( + Effect.filterOrFail( + (observation) => observation.model === input.model, + () => new LiveEvalCliError({ reason: "trial-failed" }), + ), + Effect.mapError(() => new LiveEvalCliError({ reason: "trial-failed" })), + ); + } + }, + ); + const encoded = yield* encodePrettyUnknownJson(report).pipe( + Effect.map((json) => `${json}\n`), + Effect.mapError(() => new LiveEvalCliError({ reason: "report-write-failed" })), + ); + let capture: PublicEvalAttemptCapture | undefined; + if (options.attemptsOutputPath !== undefined) { + if (attempts === null) { + return yield* new LiveEvalCliError({ reason: "report-write-failed" }); + } + capture = yield* makePublicEvalAttemptCapture({ + runId: report.runId, + reportContent: encoded, + attempts, + }); } - capture = yield* makePublicEvalAttemptCapture({ - runId: report.runId, - reportContent: encoded, - attempts, - }); - } - yield* writeReport(outputPath, encoded); - if (options.attemptsOutputPath !== undefined && capture !== undefined) { - yield* writePublicEvalAttemptCapture({ - outputPath: options.attemptsOutputPath, - reportPath: outputPath, - capture, - }); - } - yield* Console.log( - `sanitized eval report written (${report.aggregate.overall.passed}/${report.aggregate.overall.total} passed)`, - ); - return report.aggregate.overall.passed === report.aggregate.overall.total ? 0 : 2; - }); + yield* writeReport(outputPath, encoded); + if (options.attemptsOutputPath !== undefined && capture !== undefined) { + yield* writePublicEvalAttemptCapture({ + outputPath: options.attemptsOutputPath, + reportPath: outputPath, + capture, + }); + } + yield* Console.log( + `sanitized eval report written (${report.aggregate.overall.passed}/${report.aggregate.overall.total} passed)`, + ); + return report.aggregate.overall.passed === report.aggregate.overall.total ? 0 : 2; + }), + ); -const program = parseOptions(process.argv.slice(2)).pipe( - Effect.flatMap(run), +const program = parseLiveEvalCliOptions(process.argv.slice(2)).pipe( + Effect.flatMap((parsed) => + parsed.mode === "help" ? Console.log(parsed.usage).pipe(Effect.as(0)) : run(parsed.options), + ), Effect.matchEffect({ - onFailure: (error) => - Console.error( - errorTag(error) === "LiveEvalCliError" && - typeof error === "object" && - error !== null && - Reflect.get(error, "reason") === "invalid-arguments" - ? usage - : `live eval failed (${errorTag(error)})`, - ).pipe(Effect.as(1)), + onFailure: (error) => Console.error(formatLiveEvalCliFailure(error)).pipe(Effect.as(1)), onSuccess: Effect.succeed, }), Effect.tap((exitCode) => @@ -715,4 +1162,6 @@ const main = Layer.build(Layer.mergeAll(BunServices.layer, BunHttpClient.layer)) Effect.scoped, ); -BunRuntime.runMain(main); +if (import.meta.main) { + BunRuntime.runMain(main); +} diff --git a/packages/evals/src/claude-cli.ts b/packages/evals/src/claude-cli.ts new file mode 100644 index 0000000..6a55f61 --- /dev/null +++ b/packages/evals/src/claude-cli.ts @@ -0,0 +1,1103 @@ +import { + type GinaReadToolName, + listCatalogToolNames, + PRODUCTION_MCP_URL, + READ_SCOPE, + SKILL_NAMES, +} from "@askgina/contracts"; +import { + Clock, + Data, + DateTime, + Duration, + Effect, + FileSystem, + Function, + Path, + Redacted, + Stream, +} from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; + +import type { + PluginEvalCase, + PluginEvalObservation, + PluginEvalTokenUsage, + PluginEvalToolCall, +} from "./contracts"; +import { collectBoundedUtf8Output } from "./bounded-output"; + +// Claude Code print-mode contract assumed by this adapter: +// @anthropic-ai/claude-code >= 2.1.259 for --permission-prompts, >= 2.1.248 for +// --restricted, plus --strict-mcp-config/--no-session-persistence. +// --max-turns is documented for -p even when omitted from `claude --help`. +// Flag set checked against v2.1.263 help and official CLI/headless docs. +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_MAX_TURNS = 8; +const MAXIMUM_MAX_TURNS = 32; +const PROCESS_FORCE_KILL_AFTER = Duration.seconds(1); +const ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY"; +const ASK_GINA_MCP_SERVER = "ask-gina"; +const claudeMcpToolName = (canonicalName: string): string => + `mcp__${ASK_GINA_MCP_SERVER.replace(/[^a-zA-Z0-9_-]/g, "_")}__${canonicalName.replace(/[^a-zA-Z0-9_-]/g, "_")}`; +const CANONICAL_CATALOG_TOOL_NAMES = listCatalogToolNames(); +const CLAUDE_MCP_TOOL_NAME_LOOKUP: Readonly> = (() => { + const lookup: Record = {}; + for (const canonicalName of CANONICAL_CATALOG_TOOL_NAMES) { + const nativeName = claudeMcpToolName(canonicalName); + if (Object.hasOwn(lookup, nativeName)) { + throw new Error("Gina MCP catalogue has colliding Claude native tool names"); + } + lookup[nativeName] = canonicalName; + } + return lookup; +})(); +const CLAUDE_INIT_TOOL_NAMES: readonly string[] = [ + "Skill", + "Read", + ...CANONICAL_CATALOG_TOOL_NAMES.map(claudeMcpToolName), +]; +const CLAUDE_CONFIG_DIR_ENV = "CLAUDE_CONFIG_DIR"; +const MCP_DISCOVERY_CACHE_ENV = "MCP_DISCOVERY_CACHE"; +const UTF8_ENCODER = new TextEncoder(); + +export const CLAUDE_CLI_MAX_STDOUT_BYTES = 1_048_576; +export const CLAUDE_CLI_MAX_STDERR_BYTES = 65_536; + +const ASK_GINA_SKILL_NAME_SET: Readonly> = { + "review-gina-account": true, + "research-spot-tokens": true, + "research-hyperliquid": true, + "research-prediction-markets": true, +}; +const CLAUDE_SKILL_PERMISSION_RULES = SKILL_NAMES.map( + (skill) => `Skill(${ASK_GINA_MCP_SERVER}:${skill})`, +); + +const CLAUDE_EFFORTS = { + low: true, + medium: true, + high: true, + xhigh: true, + max: true, +} as const; + +type ClaudeEffort = keyof typeof CLAUDE_EFFORTS; + +const CLAUDE_NON_ACTION_BLOCK_TYPES: Readonly> = { + text: true, + thinking: true, + redacted_thinking: true, +}; + +const DISALLOWED_BUILTIN_TOOLS = + "Bash,Write,Edit,NotebookEdit,WebFetch,WebSearch,Agent,Task,Glob,Grep,LS"; + +export const CLAUDE_CLI_ALLOWED_ENVIRONMENT_NAMES = [ + "PATH", + "HOME", + "USERPROFILE", + "SystemRoot", + "WINDIR", + "COMSPEC", + "PATHEXT", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TERM", + "COLORTERM", + "NO_COLOR", + "TMPDIR", + "TMP", + "TEMP", + "XDG_CONFIG_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "XDG_RUNTIME_DIR", + "CI", +] as const; + +export class PluginEvalClaudeCliSpawnError extends Data.TaggedError( + "PluginEvalClaudeCliSpawnError", +)<{ + readonly caseId: string; + readonly reason: + | "catalog-mismatch" + | "could_not_collect_output" + | "could_not_start" + | "could_not_write_config" + | "invalid-options"; +}> {} + +export class PluginEvalClaudeCliTimeoutError extends Data.TaggedError( + "PluginEvalClaudeCliTimeoutError", +)<{ + readonly caseId: string; + readonly timeoutMs: number; +}> {} + +export class PluginEvalClaudeCliProcessError extends Data.TaggedError( + "PluginEvalClaudeCliProcessError", +)<{ + readonly caseId: string; + readonly reason: + | "incomplete-stream" + | "malformed-jsonl" + | "nonzero-exit" + | "stderr-truncated" + | "stdout-truncated"; +}> {} + +export type PluginEvalClaudeCliError = + | PluginEvalClaudeCliSpawnError + | PluginEvalClaudeCliTimeoutError + | PluginEvalClaudeCliProcessError; + +export interface ClaudeCliParsedStream { + readonly activated_skills: readonly string[]; + readonly tool_calls: readonly PluginEvalToolCall[]; + readonly unsupported_actions: number; + readonly malformed_jsonl: boolean; + readonly incomplete: boolean; + readonly available_tools?: readonly string[]; + readonly final_answer?: string; + readonly token_usage?: PluginEvalTokenUsage; + readonly error?: string; +} + +export interface ClaudeCliCommand { + readonly caseId: string; + readonly command: string; + readonly args: readonly string[]; + readonly workingDirectory: string; + readonly environment: Readonly>; + readonly stdoutLimitBytes: number; + readonly stderrLimitBytes: number; +} + +export interface ClaudeCliProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stdoutTruncated: boolean; + readonly stderrTruncated: boolean; +} + +export interface ClaudeCliTrialRunner { + readonly run: ( + command: ClaudeCliCommand, + ) => Effect.Effect< + ClaudeCliProcessResult, + PluginEvalClaudeCliSpawnError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path + >; +} + +export interface ClaudeCliTrialOptions { + readonly runId: string; + readonly repetition: number; + readonly availableTools: readonly string[]; + readonly workingDirectory: string; + readonly executablePath: string; + readonly pluginDirectory: string; + readonly mcpAuthorization: Redacted.Redacted; + readonly apiKey: Redacted.Redacted; + readonly model: string; + readonly reasoning: string; + readonly parentEnvironment: Readonly>; + readonly timeoutMs?: number; + readonly maxTurns?: number; + readonly runner?: ClaudeCliTrialRunner; +} + +export interface ClaudeCliParseContext { + readonly pluginDirectory: string; + readonly workingDirectory: string; + readonly forbiddenReadRoots: readonly string[]; +} + +interface ClaudeCliPreparedPaths { + readonly configDirectory: string; + readonly mcpConfigPath: string; + readonly settingsPath: string; + readonly pluginDirectory: string; + readonly workingDirectory: string; +} + +interface ValidatedClaudeCliTrialOptions { + readonly runId: string; + readonly repetition: number; + readonly availableTools: readonly string[]; + readonly workingDirectory: string; + readonly executablePath: string; + readonly pluginDirectory: string; + readonly mcpAuthorization: string; + readonly apiKey: string; + readonly model: string; + readonly reasoning: ClaudeEffort; + readonly parentEnvironment: Readonly>; + readonly timeoutMs: number; + readonly maxTurns: number; +} + +const isJsonObject = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const asString = (value: unknown): string | undefined => + typeof value === "string" && value.length > 0 ? value : undefined; + +const isClaudeEffort = (value: string): value is ClaudeEffort => + Object.hasOwn(CLAUDE_EFFORTS, value); + +const isAskGinaSkillName = (value: string): value is (typeof SKILL_NAMES)[number] => + Object.hasOwn(ASK_GINA_SKILL_NAME_SET, value); + +const catalogsMatch = (left: readonly string[], right: readonly string[]): boolean => { + if (left.length !== right.length) return false; + const uniqueLeft = new Set(left); + return uniqueLeft.size === left.length && right.every((tool) => uniqueLeft.has(tool)); +}; + +const collapsePath = (value: string): string => { + const replaced = value.replaceAll("\\", "/"); + const absolute = replaced.startsWith("/"); + const resolved: string[] = []; + for (const segment of replaced.split("/")) { + if (segment === "" || segment === ".") continue; + if (segment === "..") { + resolved.pop(); + continue; + } + resolved.push(segment); + } + return `${absolute ? "/" : ""}${resolved.join("/")}`; +}; + +const isWithin = (parent: string, child: string): boolean => { + const normalizedParent = collapsePath(parent).replace(/\/$/u, ""); + const normalizedChild = collapsePath(child); + return normalizedChild === normalizedParent || normalizedChild.startsWith(`${normalizedParent}/`); +}; + +const resolveCandidatePath = (workingDirectory: string, candidate: string): string => { + const normalized = candidate.replaceAll("\\", "/"); + if (normalized.startsWith("/")) return collapsePath(normalized); + return collapsePath(`${workingDirectory.replace(/\/$/u, "")}/${normalized}`); +}; + +const absoluteReadRule = (target: string): string => { + const collapsed = collapsePath(target).replace(/\/$/u, ""); + const withoutLeading = collapsed.startsWith("/") ? collapsed.slice(1) : collapsed; + return `Read(//${withoutLeading}/**)`; +}; + +const mcpAuthorizationHeader = (token: string): string => + token.startsWith("Bearer ") ? token : `Bearer ${token}`; + +const promptFromCase = (evalCase: PluginEvalCase): string => { + const userTurns = evalCase.turns + .filter((turn) => turn.role === "user") + .map((turn) => turn.content); + return (userTurns.length > 0 ? userTurns : evalCase.turns.map((turn) => turn.content)).join( + "\n\n", + ); +}; + +export const buildClaudeCliEnvironment = ( + parentEnvironment: Readonly>, +): Record => { + const environment: Record = {}; + for (const name of CLAUDE_CLI_ALLOWED_ENVIRONMENT_NAMES) { + const value = parentEnvironment[name]; + if (value !== undefined) environment[name] = value; + } + return environment; +}; + +const claudeEvalSettings = ( + paths: ClaudeCliPreparedPaths, + availableTools: readonly string[], +): string => + JSON.stringify({ + disableAllHooks: true, + disableBundledSkills: true, + disableAutoMode: "disable", + disableBypassPermissionsMode: "disable", + permissions: { + defaultMode: "dontAsk", + disableAutoMode: "disable", + disableBypassPermissionsMode: "disable", + blockReadsOutsideWorkingDirectories: true, + allow: [ + ...CLAUDE_SKILL_PERMISSION_RULES, + absoluteReadRule(paths.pluginDirectory), + absoluteReadRule(pathJoin(paths.pluginDirectory, "skills")), + ...availableTools.map(claudeMcpToolName), + ], + deny: [ + "Bash", + "Write", + "Edit", + "NotebookEdit", + "WebFetch", + "WebSearch", + "Agent", + "Task", + absoluteReadRule(paths.configDirectory), + ], + }, + }); + +const claudeMcpConfig = (authorization: string): string => + JSON.stringify({ + mcpServers: { + [ASK_GINA_MCP_SERVER]: { + type: "http", + url: PRODUCTION_MCP_URL, + headers: { + Authorization: mcpAuthorizationHeader(authorization), + }, + }, + }, + }); + +const writeClaudeEvalConfig = ( + caseId: string, + options: ValidatedClaudeCliTrialOptions, + paths: ClaudeCliPreparedPaths, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.chmod(paths.configDirectory, 0o700); + yield* fs.makeDirectory(pathJoin(paths.configDirectory, "tmp"), { + recursive: true, + mode: 0o700, + }); + yield* fs.writeFileString( + paths.settingsPath, + claudeEvalSettings(paths, options.availableTools), + { + flag: "w", + mode: 0o600, + }, + ); + yield* fs.writeFileString(paths.mcpConfigPath, claudeMcpConfig(options.mcpAuthorization), { + flag: "w", + mode: 0o600, + }); + }).pipe( + Effect.mapError( + () => + new PluginEvalClaudeCliSpawnError({ + caseId, + reason: "could_not_write_config", + }), + ), + ); + +const pathJoin = (left: string, right: string): string => + `${left.replace(/[/\\]+$/u, "")}/${right}`; + +const makeClaudeCliCommand = ( + evalCase: PluginEvalCase, + options: ValidatedClaudeCliTrialOptions, + paths: ClaudeCliPreparedPaths, +): ClaudeCliCommand => { + const configTmp = pathJoin(paths.configDirectory, "tmp"); + const addDirectories = [paths.pluginDirectory, pathJoin(paths.pluginDirectory, "skills")] + .filter((root) => !isWithin(paths.workingDirectory, root)) + .flatMap((root) => ["--add-dir", root]); + const allowedTools = [ + ...CLAUDE_SKILL_PERMISSION_RULES, + "Read", + ...options.availableTools.map(claudeMcpToolName), + ].join(","); + return { + caseId: evalCase.id, + command: options.executablePath, + args: [ + "-p", + "--output-format", + "stream-json", + "--verbose", + "--restricted", + "--strict-mcp-config", + "--permission-prompts", + "none", + "--permission-mode", + "dontAsk", + "--no-session-persistence", + "--plugin-dir", + paths.pluginDirectory, + "--mcp-config", + paths.mcpConfigPath, + "--settings", + paths.settingsPath, + ...addDirectories, + "--tools", + "Skill,Read", + "--disallowedTools", + DISALLOWED_BUILTIN_TOOLS, + "--allowedTools", + allowedTools, + "--model", + options.model, + "--effort", + options.reasoning, + "--max-turns", + String(options.maxTurns), + "--", + promptFromCase(evalCase), + ], + workingDirectory: paths.workingDirectory, + environment: { + ...buildClaudeCliEnvironment(options.parentEnvironment), + HOME: paths.configDirectory, + USERPROFILE: paths.configDirectory, + [CLAUDE_CONFIG_DIR_ENV]: paths.configDirectory, + XDG_CONFIG_HOME: paths.configDirectory, + XDG_CACHE_HOME: paths.configDirectory, + XDG_DATA_HOME: paths.configDirectory, + TMPDIR: configTmp, + TMP: configTmp, + TEMP: configTmp, + CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + DISABLE_AUTOUPDATER: "1", + DISABLE_TELEMETRY: "1", + DISABLE_ERROR_REPORTING: "1", + CLAUDE_CODE_DISABLE_TERMINAL_TITLE: "1", + CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", + CLAUDE_CODE_MAX_RETRIES: "0", + CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK: "1", + [MCP_DISCOVERY_CACHE_ENV]: "0", + [ANTHROPIC_API_KEY_ENV]: options.apiKey, + }, + stdoutLimitBytes: CLAUDE_CLI_MAX_STDOUT_BYTES, + stderrLimitBytes: CLAUDE_CLI_MAX_STDERR_BYTES, + }; +}; + +const detectClaudeCliOutputTruncation = ( + stream: Stream.Stream, + maximumBytes: number, +): Effect.Effect => { + const limit = Math.max(0, Math.trunc(maximumBytes)); + return stream.pipe( + Stream.runFold( + () => ({ byteLength: 0, truncated: false }), + (output, chunk) => { + const remaining = Math.max(0, limit - output.byteLength); + output.byteLength += Math.min(remaining, chunk.byteLength); + if (chunk.byteLength > remaining) output.truncated = true; + return output; + }, + ), + Effect.map((output) => output.truncated), + ); +}; + +export const effectClaudeCliTrialRunner: ClaudeCliTrialRunner = { + run: (command) => + Effect.scoped( + Effect.gen(function* () { + const process = yield* ChildProcess.make(command.command, command.args, { + cwd: command.workingDirectory, + env: { ...command.environment }, + extendEnv: false, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + forceKillAfter: PROCESS_FORCE_KILL_AFTER, + }).pipe( + Effect.mapError( + () => + new PluginEvalClaudeCliSpawnError({ + caseId: command.caseId, + reason: "could_not_start", + }), + ), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + collectBoundedUtf8Output(process.stdout, command.stdoutLimitBytes), + detectClaudeCliOutputTruncation(process.stderr, command.stderrLimitBytes), + process.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.mapError( + () => + new PluginEvalClaudeCliSpawnError({ + caseId: command.caseId, + reason: "could_not_collect_output", + }), + ), + ); + return { + exitCode, + stdout: stdout.text, + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr, + } satisfies ClaudeCliProcessResult; + }), + ), +}; + +const messageContent = (event: Readonly>): unknown => { + if (isJsonObject(event.message)) return event.message.content; + return event.content; +}; + +const contentBlocks = (value: unknown): readonly Record[] => { + if (!Array.isArray(value)) return []; + return value.filter(isJsonObject); +}; + +const skillFromUnknown = (value: unknown): string | undefined => { + const raw = asString(value)?.trim(); + if (raw === undefined) return undefined; + const trimmed = raw.replace(/^\/+/u, ""); + const separator = trimmed.indexOf(":"); + const skill = + separator === -1 + ? trimmed + : trimmed.slice(0, separator) === ASK_GINA_MCP_SERVER + ? trimmed.slice(separator + 1) + : trimmed; + return isAskGinaSkillName(skill) ? skill : undefined; +}; + +const skillFromToolInput = (input: Readonly>): string | undefined => + skillFromUnknown(input.skill) ?? skillFromUnknown(input.name) ?? skillFromUnknown(input.command); + +const skillFromPluginPath = (candidate: string, pluginDirectory: string): string | undefined => { + const normalizedCandidate = collapsePath(candidate); + const skillsRoot = collapsePath(pathJoin(pluginDirectory, "skills")).replace(/\/$/u, ""); + for (const skill of SKILL_NAMES) { + if (normalizedCandidate === `${skillsRoot}/${skill}/SKILL.md`) { + return skill; + } + } + return undefined; +}; + +const catalogFromClaudeInit = ( + event: Readonly>, + pluginDirectory: string, +): readonly GinaReadToolName[] | undefined => { + const pluginErrors = event.plugin_errors; + const serverErrors = event.mcp_server_errors; + if ( + (pluginErrors !== undefined && (!Array.isArray(pluginErrors) || pluginErrors.length > 0)) || + (serverErrors !== undefined && (!Array.isArray(serverErrors) || serverErrors.length > 0)) + ) { + return undefined; + } + const plugins = event.plugins; + if (!Array.isArray(plugins) || plugins.length !== 1 || !isJsonObject(plugins[0])) + return undefined; + const plugin = plugins[0]; + const pluginPath = asString(plugin.path); + if ( + asString(plugin.name) !== ASK_GINA_MCP_SERVER || + pluginPath === undefined || + collapsePath(pluginPath) !== collapsePath(pluginDirectory) + ) { + return undefined; + } + const servers = event.mcp_servers; + if (!Array.isArray(servers) || servers.length !== 1 || !isJsonObject(servers[0])) + return undefined; + const server = servers[0]; + if (asString(server.name) !== ASK_GINA_MCP_SERVER || asString(server.status) !== "connected") { + return undefined; + } + const tools = event.tools; + if (!Array.isArray(tools)) return undefined; + const toolNames: string[] = []; + let endConversationSeen = false; + for (const tool of tools) { + if (typeof tool !== "string" || tool.length === 0) return undefined; + if (tool === "EndConversation") { + if (endConversationSeen) return undefined; + endConversationSeen = true; + continue; + } + toolNames.push(tool); + } + if (!catalogsMatch(toolNames, CLAUDE_INIT_TOOL_NAMES)) return undefined; + return CANONICAL_CATALOG_TOOL_NAMES; +}; + +const decodeToolArguments = (value: unknown): PluginEvalToolCall["arguments"] | undefined => { + if (typeof value === "string") { + try { + const parsed: unknown = JSON.parse(value); + return isJsonObject(parsed) ? (parsed as PluginEvalToolCall["arguments"]) : undefined; + } catch { + return undefined; + } + } + return isJsonObject(value) ? (value as PluginEvalToolCall["arguments"]) : undefined; +}; + +const tokenUsageFromUnknown = (value: unknown): PluginEvalTokenUsage | undefined => { + if (!isJsonObject(value)) return undefined; + const inputTokens = value.input_tokens; + const outputTokens = value.output_tokens; + if ( + typeof inputTokens !== "number" || + !Number.isSafeInteger(inputTokens) || + inputTokens < 0 || + typeof outputTokens !== "number" || + !Number.isSafeInteger(outputTokens) || + outputTokens < 0 + ) { + return undefined; + } + const totalTokens = + value.total_tokens === undefined ? inputTokens + outputTokens : value.total_tokens; + if (typeof totalTokens !== "number" || !Number.isSafeInteger(totalTokens) || totalTokens < 0) { + return undefined; + } + return { + input_tokens: inputTokens, + output_tokens: outputTokens, + total_tokens: totalTokens, + }; +}; + +const serializedResultBytes = (value: unknown): number | undefined => { + if (value === undefined) return undefined; + const serialized = typeof value === "string" ? value : JSON.stringify(value); + return UTF8_ENCODER.encode(serialized).byteLength; +}; + +const classifyToolUse = ( + name: string, + input: Readonly>, + context: ClaudeCliParseContext, +): + | { readonly kind: "skill"; readonly skill: string } + | { readonly kind: "read"; readonly skill?: string } + | { readonly kind: "mcp"; readonly name: string } + | { readonly kind: "unsupported" } => { + if (name === "Skill") { + const skill = skillFromToolInput(input); + return skill === undefined ? { kind: "unsupported" } : { kind: "skill", skill }; + } + if (name === "Read") { + const candidate = asString(input.file_path) ?? asString(input.path) ?? asString(input.file); + if (candidate === undefined) return { kind: "unsupported" }; + const resolved = resolveCandidatePath(context.workingDirectory, candidate); + if (context.forbiddenReadRoots.some((root) => isWithin(root, resolved))) { + return { kind: "unsupported" }; + } + const skill = skillFromPluginPath(resolved, context.pluginDirectory); + if (skill !== undefined) return { kind: "read", skill }; + if ( + isWithin(context.pluginDirectory, resolved) || + isWithin(context.workingDirectory, resolved) + ) { + return { kind: "read" }; + } + return { kind: "unsupported" }; + } + const canonicalName = CLAUDE_MCP_TOOL_NAME_LOOKUP[name]; + if (canonicalName === undefined) return { kind: "unsupported" }; + return { kind: "mcp", name: canonicalName }; +}; + +export const parseClaudeCliStreamJson = Function.dual< + (context: ClaudeCliParseContext) => (jsonl: string) => ClaudeCliParsedStream, + (jsonl: string, context: ClaudeCliParseContext) => ClaudeCliParsedStream +>(2, (jsonl, context) => { + const activatedSkills: string[] = []; + const toolCalls: PluginEvalToolCall[] = []; + let malformedJsonl = false; + const pending = new Map< + string, + | { + readonly kind: "mcp"; + readonly sequence: number; + readonly name: string; + readonly arguments: PluginEvalToolCall["arguments"]; + } + | { readonly kind: "skill"; readonly skill: string } + | { readonly kind: "read"; readonly skill?: string } + | { readonly kind: "ignored" } + >(); + const seenToolUseIds = new Set(); + let finalAnswer: string | undefined; + let tokenUsage: PluginEvalTokenUsage | undefined; + let unsupportedActions = 0; + let resultSeen = false; + let resultSuccess = false; + let eventsAfterResult = 0; + let nextToolCallSequence = 0; + let initSeen = false; + let initInvalid = false; + let sawActionOrResult = false; + let initAvailableTools: readonly GinaReadToolName[] | undefined; + + const rememberSkill = (skill: string): void => { + if (!activatedSkills.includes(skill)) activatedSkills.push(skill); + }; + + for (const rawLine of jsonl.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0) continue; + if (resultSeen) eventsAfterResult += 1; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + malformedJsonl = true; + continue; + } + if (!isJsonObject(parsed)) { + malformedJsonl = true; + continue; + } + + const type = asString(parsed.type); + if (type === "system") { + if (asString(parsed.subtype) !== "init") continue; + if (initSeen || sawActionOrResult) { + initInvalid = true; + initAvailableTools = undefined; + continue; + } + initSeen = true; + const catalog = catalogFromClaudeInit(parsed, context.pluginDirectory); + if (catalog === undefined) { + initInvalid = true; + continue; + } + initAvailableTools = catalog; + continue; + } + if (type === "assistant" || type === "user" || type === "result") { + if (initAvailableTools === undefined) { + sawActionOrResult = true; + initInvalid = true; + continue; + } + } + if (type === "assistant") { + for (const block of contentBlocks(messageContent(parsed))) { + const blockType = asString(block.type); + if (blockType === undefined) continue; + if (Object.hasOwn(CLAUDE_NON_ACTION_BLOCK_TYPES, blockType)) continue; + if (blockType !== "tool_use") { + unsupportedActions += 1; + continue; + } + const id = asString(block.id); + const name = asString(block.name); + const input = isJsonObject(block.input) ? block.input : undefined; + if (id === undefined || name === undefined) { + malformedJsonl = true; + continue; + } + if (seenToolUseIds.has(id)) { + malformedJsonl = true; + continue; + } + seenToolUseIds.add(id); + const classified = classifyToolUse(name, input ?? {}, context); + if (classified.kind === "unsupported") { + unsupportedActions += 1; + pending.set(id, { kind: "ignored" }); + continue; + } + if (classified.kind === "skill") { + pending.set(id, { kind: "skill", skill: classified.skill }); + continue; + } + if (classified.kind === "read") { + pending.set( + id, + classified.skill === undefined + ? { kind: "read" } + : { kind: "read", skill: classified.skill }, + ); + continue; + } + const sequence = nextToolCallSequence; + nextToolCallSequence += 1; + const argumentsValue = decodeToolArguments(input); + if (argumentsValue === undefined) { + toolCalls.push({ + sequence, + name: classified.name, + arguments: {}, + requested_scope: READ_SCOPE, + error: { + code: "invalid_arguments", + message: "Claude returned invalid MCP arguments", + }, + }); + pending.set(id, { kind: "ignored" }); + } else { + pending.set(id, { + kind: "mcp", + sequence, + name: classified.name, + arguments: argumentsValue, + }); + } + } + continue; + } + + if (type === "user") { + for (const block of contentBlocks(messageContent(parsed))) { + if (asString(block.type) !== "tool_result") continue; + const toolUseId = asString(block.tool_use_id) ?? asString(block.toolUseId); + if (toolUseId === undefined) { + malformedJsonl = true; + continue; + } + const pendingCall = pending.get(toolUseId); + pending.delete(toolUseId); + if (pendingCall === undefined) { + malformedJsonl = true; + continue; + } + const failed = block.is_error === true || block.isError === true; + if (pendingCall.kind === "skill" || pendingCall.kind === "read") { + if (!failed && pendingCall.skill !== undefined) rememberSkill(pendingCall.skill); + continue; + } + if (pendingCall.kind !== "mcp") continue; + const resultBytes = serializedResultBytes(block.content ?? block.output); + toolCalls.push({ + sequence: pendingCall.sequence, + name: pendingCall.name, + arguments: pendingCall.arguments, + requested_scope: READ_SCOPE, + ...(resultBytes === undefined ? {} : { result_bytes: resultBytes }), + ...(failed ? { error: { message: "MCP tool call failed" } } : {}), + }); + } + continue; + } + + if (type === "result") { + resultSeen = true; + const subtype = asString(parsed.subtype); + resultSuccess = subtype === "success" && parsed.is_error === false; + const resultText = asString(parsed.result); + if (resultText !== undefined) finalAnswer = resultText; + tokenUsage = tokenUsageFromUnknown(parsed.usage) ?? tokenUsage; + continue; + } + } + + const incomplete = + initAvailableTools === undefined || + initInvalid || + !resultSuccess || + [...pending.values()].some((value) => value.kind !== "ignored") || + eventsAfterResult > 0; + const orderedToolCalls = [...toolCalls].sort((left, right) => left.sequence - right.sequence); + return { + activated_skills: activatedSkills, + tool_calls: orderedToolCalls, + unsupported_actions: unsupportedActions, + malformed_jsonl: malformedJsonl, + incomplete, + ...(initAvailableTools === undefined ? {} : { available_tools: initAvailableTools }), + ...(finalAnswer === undefined ? {} : { final_answer: finalAnswer }), + ...(tokenUsage === undefined ? {} : { token_usage: tokenUsage }), + ...(unsupportedActions === 0 + ? {} + : { + error: "Claude used an action outside Skill, plugin reads, or canonical Gina MCP reads", + }), + }; +}); + +const validateOptions = ( + evalCase: PluginEvalCase, + options: ClaudeCliTrialOptions, +): Effect.Effect => { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const maxTurns = options.maxTurns ?? DEFAULT_MAX_TURNS; + const apiKey = Redacted.value(options.apiKey); + const mcpAuthorization = Redacted.value(options.mcpAuthorization); + if (!catalogsMatch(options.availableTools, listCatalogToolNames())) { + return Effect.fail( + new PluginEvalClaudeCliSpawnError({ + caseId: evalCase.id, + reason: "catalog-mismatch", + }), + ); + } + if ( + options.runId.trim().length === 0 || + options.model.trim().length === 0 || + apiKey.trim().length === 0 || + mcpAuthorization.trim().length === 0 || + !options.executablePath.startsWith("/") || + !options.workingDirectory.startsWith("/") || + !options.pluginDirectory.startsWith("/") || + !Number.isSafeInteger(options.repetition) || + options.repetition <= 0 || + !Number.isSafeInteger(timeoutMs) || + timeoutMs <= 0 || + !Number.isSafeInteger(maxTurns) || + maxTurns <= 0 || + maxTurns > MAXIMUM_MAX_TURNS || + !isClaudeEffort(options.reasoning) + ) { + return Effect.fail( + new PluginEvalClaudeCliSpawnError({ + caseId: evalCase.id, + reason: "invalid-options", + }), + ); + } + return Effect.succeed({ + runId: options.runId, + repetition: options.repetition, + availableTools: options.availableTools, + workingDirectory: options.workingDirectory, + executablePath: options.executablePath, + pluginDirectory: options.pluginDirectory, + mcpAuthorization, + apiKey, + model: options.model, + reasoning: options.reasoning, + parentEnvironment: options.parentEnvironment, + timeoutMs, + maxTurns, + }); +}; + +export const runClaudeCliPluginEvalTrial = Function.dual< + ( + options: ClaudeCliTrialOptions, + ) => ( + evalCase: PluginEvalCase, + ) => Effect.Effect< + PluginEvalObservation, + PluginEvalClaudeCliError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path + >, + ( + evalCase: PluginEvalCase, + options: ClaudeCliTrialOptions, + ) => Effect.Effect< + PluginEvalObservation, + PluginEvalClaudeCliError, + ChildProcessSpawner | FileSystem.FileSystem | Path.Path + > +>(2, (evalCase, options) => { + const runner = options.runner ?? effectClaudeCliTrialRunner; + return Effect.scoped( + Effect.gen(function* () { + const validated = yield* validateOptions(evalCase, options); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const configDirectory = yield* fs + .makeTempDirectoryScoped({ prefix: "ask-gina-claude-config-" }) + .pipe( + Effect.mapError( + () => + new PluginEvalClaudeCliSpawnError({ + caseId: evalCase.id, + reason: "could_not_write_config", + }), + ), + ); + const paths: ClaudeCliPreparedPaths = { + configDirectory, + mcpConfigPath: path.join(configDirectory, "mcp.json"), + settingsPath: path.join(configDirectory, "settings.json"), + pluginDirectory: validated.pluginDirectory, + workingDirectory: validated.workingDirectory, + }; + yield* writeClaudeEvalConfig(evalCase.id, validated, paths); + const command = makeClaudeCliCommand(evalCase, validated, paths); + const startedAt = DateTime.formatIso(yield* DateTime.now); + const startedMillis = yield* Clock.currentTimeMillis; + const processResult = yield* runner.run(command).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(validated.timeoutMs), + orElse: () => + Effect.fail( + new PluginEvalClaudeCliTimeoutError({ + caseId: evalCase.id, + timeoutMs: validated.timeoutMs, + }), + ), + }), + ); + if (processResult.stdoutTruncated) { + return yield* new PluginEvalClaudeCliProcessError({ + caseId: evalCase.id, + reason: "stdout-truncated", + }); + } + if (processResult.stderrTruncated) { + return yield* new PluginEvalClaudeCliProcessError({ + caseId: evalCase.id, + reason: "stderr-truncated", + }); + } + if (processResult.exitCode !== 0) { + return yield* new PluginEvalClaudeCliProcessError({ + caseId: evalCase.id, + reason: "nonzero-exit", + }); + } + + const parsed = parseClaudeCliStreamJson(processResult.stdout, { + pluginDirectory: validated.pluginDirectory, + workingDirectory: validated.workingDirectory, + forbiddenReadRoots: [configDirectory], + }); + if (parsed.malformed_jsonl) { + return yield* new PluginEvalClaudeCliProcessError({ + caseId: evalCase.id, + reason: "malformed-jsonl", + }); + } + if (parsed.incomplete) { + return yield* new PluginEvalClaudeCliProcessError({ + caseId: evalCase.id, + reason: "incomplete-stream", + }); + } + const finishedMillis = yield* Clock.currentTimeMillis; + return { + version: 1, + run_id: validated.runId, + case_id: evalCase.id, + target: "claude_cli", + model: validated.model, + repetition: validated.repetition, + started_at: startedAt, + status: parsed.error === undefined ? "completed" : "failed", + duration_ms: Math.max(0, finishedMillis - startedMillis), + activated_skills: [...parsed.activated_skills], + tool_calls: [...parsed.tool_calls], + ...(parsed.available_tools === undefined + ? {} + : { available_tools: [...parsed.available_tools] }), + ...(parsed.token_usage === undefined ? {} : { token_usage: parsed.token_usage }), + ...(parsed.final_answer === undefined ? {} : { final_answer: parsed.final_answer }), + ...(parsed.error === undefined ? {} : { error: parsed.error }), + } satisfies PluginEvalObservation; + }), + ).pipe( + Effect.withSpan("plugin_evals.claude_cli_trial", { + attributes: { + "plugin_eval.case_id": evalCase.id, + "plugin_eval.repetition": options.repetition, + }, + }), + ); +}); diff --git a/packages/evals/src/contracts.ts b/packages/evals/src/contracts.ts index aa25245..088bd85 100644 --- a/packages/evals/src/contracts.ts +++ b/packages/evals/src/contracts.ts @@ -20,10 +20,13 @@ export const PluginEvalCategorySchema = Schema.Literals([ export const PluginEvalTargetSchema = Schema.Literals([ "fixture", "responses_api", + "openrouter_api", "chatgpt_developer", "chatgpt_plugin", "browser_replay", "codex_cli", + "claude_cli", + "omp_harness", ]); export const PluginEvalTurnSchema = Schema.Struct({ @@ -238,6 +241,7 @@ export const PluginEvalReplayReportSchema = Schema.Struct({ scores: Schema.Array(PluginEvalCaseScoreSchema).check(Schema.isMinLength(1)), }); +export type PluginEvalTarget = typeof PluginEvalTargetSchema.Type; export type PluginEvalCase = typeof PluginEvalCaseSchema.Type; export type PluginEvalSuite = typeof PluginEvalSuiteSchema.Type; export type PluginEvalRunManifest = typeof PluginEvalRunManifestSchema.Type; diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts index 94e25f4..ec64442 100644 --- a/packages/evals/src/index.ts +++ b/packages/evals/src/index.ts @@ -41,6 +41,15 @@ export { type PluginEvalResponsesError, type ResponsesApiTrialOptions, } from "./responses-api"; +export { + PluginEvalOpenRouterGenerationError, + PluginEvalOpenRouterMcpError, + PluginEvalOpenRouterRequestError, + PluginEvalOpenRouterTimeoutError, + runOpenRouterPluginEvalTrial, + type PluginEvalOpenRouterError, + type OpenRouterTrialOptions, +} from "./openrouter"; export { attestCodexExecutable, PluginEvalCodexCliExecutableError, @@ -56,6 +65,29 @@ export { type CodexCliTrialRunner, type PluginEvalCodexCliError, } from "./codex-cli"; +export { + PluginEvalClaudeCliProcessError, + PluginEvalClaudeCliSpawnError, + PluginEvalClaudeCliTimeoutError, + runClaudeCliPluginEvalTrial, + type ClaudeCliTrialOptions, + type PluginEvalClaudeCliError, +} from "./claude-cli"; +export { + prepareOmpHarnessRuntime, + runOmpHarnessPluginEvalTrial, + PluginEvalOmpHarnessExecutableError, + PluginEvalOmpHarnessRequestError, + PluginEvalOmpHarnessSpawnError, + PluginEvalOmpHarnessMcpError, + PluginEvalOmpHarnessProcessError, + PluginEvalOmpHarnessTimeoutError, + type PrepareOmpHarnessRuntimeOptions, + type PreparedOmpHarnessRuntime, + type OmpHarnessTrialOptions, + type OmpProvider, + type PluginEvalOmpHarnessError, +} from "./omp-harness"; export { makeSanitizedEvalRunReport, sanitizeEvalReplay, @@ -122,6 +154,7 @@ export { PluginEvalReplayReportSchema, PluginEvalRunManifestSchema, PluginEvalSuiteSchema, + PluginEvalTargetSchema, PluginEvalToolCallSchema, type PluginEvalCase, type PluginEvalCaseScore, @@ -132,5 +165,6 @@ export { type PluginEvalObservationSet, type PluginEvalReplayReport, type PluginEvalSuite, + type PluginEvalTarget, type PluginEvalToolCall, } from "./contracts"; diff --git a/packages/evals/src/live.ts b/packages/evals/src/live.ts index d4344a5..bed330e 100644 --- a/packages/evals/src/live.ts +++ b/packages/evals/src/live.ts @@ -6,7 +6,12 @@ import { } from "@askgina/contracts"; import { Data, DateTime, Effect, Function } from "effect"; -import type { PluginEvalCase, PluginEvalObservation, PluginEvalSuite } from "./contracts"; +import type { + PluginEvalCase, + PluginEvalObservation, + PluginEvalSuite, + PluginEvalTarget, +} from "./contracts"; import type { PluginEvalObservationMismatchError } from "./grading"; import { decodePluginEvalObservationSet, @@ -36,9 +41,9 @@ export interface LiveEvalOptions { readonly captureAttempts?: boolean; readonly runId: string; readonly candidate: string; - readonly target: string; + readonly target: PluginEvalTarget; readonly model: string; - readonly displayedModel: string; + readonly displayedModel?: string; readonly reasoning: string; readonly repetitions: number; readonly accountClass: string; @@ -47,9 +52,9 @@ export interface LiveEvalOptions { export interface LiveEvalTrialInput { readonly evalCase: PluginEvalCase; readonly runId: string; - readonly target: string; + readonly target: PluginEvalTarget; readonly model: string; - readonly displayedModel: string; + readonly displayedModel?: string; readonly repetition: number; readonly startedAt: string; } @@ -185,7 +190,9 @@ export const runLiveEvalSuite = Function.dual< runId: options.runId, target: options.target, model: options.model, - displayedModel: options.displayedModel, + ...(options.displayedModel === undefined + ? {} + : { displayedModel: options.displayedModel }), repetition, startedAt: DateTime.formatIso(yield* DateTime.now), }); @@ -217,7 +224,9 @@ export const runLiveEvalSuite = Function.dual< candidate: options.candidate, target: options.target, model: options.model, - displayed_model: options.displayedModel, + ...(options.displayedModel === undefined + ? {} + : { displayed_model: options.displayedModel }), reasoning: options.reasoning, started_at: startedAt, repetitions: options.repetitions, diff --git a/packages/evals/src/load-observations.ts b/packages/evals/src/load-observations.ts index 1abb873..a287c76 100644 --- a/packages/evals/src/load-observations.ts +++ b/packages/evals/src/load-observations.ts @@ -40,12 +40,26 @@ const parseObservationYaml = ( catch: (cause) => new PluginEvalObservationSetParseError({ path, reason: String(cause) }), }); -const validateObservationSetInvariants = ( - observationSet: PluginEvalObservationSet, - path: string, -): Effect.Effect => { +export const validateObservationSetInvariants = Function.dual< + ( + path: string, + ) => ( + observationSet: PluginEvalObservationSet, + ) => Effect.Effect, + ( + observationSet: PluginEvalObservationSet, + path: string, + ) => Effect.Effect +>(2, (observationSet, path) => { const reasons: string[] = []; const seenAttempts = new Set(); + const hasValidManifestRepetitions = + Number.isSafeInteger(observationSet.manifest.repetitions) && + observationSet.manifest.repetitions > 0; + + if (!hasValidManifestRepetitions) { + reasons.push("manifest repetitions must be a positive safe integer"); + } for (const observation of observationSet.observations) { const attemptKey = `${observation.case_id}#${observation.repetition}`; @@ -63,13 +77,26 @@ const validateObservationSetInvariants = ( if (observation.model !== observationSet.manifest.model) { reasons.push(`${attemptKey} model does not match the manifest`); } - if (observation.repetition > observationSet.manifest.repetitions) { + if (observation.displayed_model !== observationSet.manifest.displayed_model) { + reasons.push(`${attemptKey} displayed_model does not match the manifest`); + } + const hasValidRepetition = + Number.isSafeInteger(observation.repetition) && observation.repetition > 0; + if (!hasValidRepetition) { + reasons.push(`${attemptKey} repetition must be a positive safe integer`); + } else if ( + hasValidManifestRepetitions && + observation.repetition > observationSet.manifest.repetitions + ) { reasons.push(`${attemptKey} exceeds the manifest repetition count`); } if (observation.status === "completed" && observation.error !== undefined) { reasons.push(`${attemptKey} is completed but has a top-level error`); } - if (observation.status !== "completed" && observation.error === undefined) { + if ( + observation.status !== "completed" && + (observation.error === undefined || observation.error.length === 0) + ) { reasons.push(`${attemptKey} is ${observation.status} but has no top-level error`); } @@ -84,7 +111,7 @@ const validateObservationSetInvariants = ( return reasons.length === 0 ? Effect.succeed(observationSet) : Effect.fail(new PluginEvalObservationSetValidationError({ path, reasons })); -}; +}); export const decodePluginEvalObservationSet = Function.dual< ( diff --git a/packages/evals/src/omp-docker-sandbox.ts b/packages/evals/src/omp-docker-sandbox.ts new file mode 100644 index 0000000..4acabab --- /dev/null +++ b/packages/evals/src/omp-docker-sandbox.ts @@ -0,0 +1,1537 @@ +import { randomBytes } from "node:crypto"; +import { createRequire } from "node:module"; +import { Duplex, PassThrough } from "node:stream"; + +import type { + HarnessV1NetworkSandboxSession, + HarnessV1PortEndpoint, + HarnessV1SandboxProvider, +} from "@ai-sdk/harness"; +import { HarnessSandboxAuthenticationError } from "@ai-sdk/harness"; +import { Data, Duration, Effect, Schema } from "effect"; + +const Dockerode: typeof import("dockerode") = createRequire(import.meta.url)("dockerode"); + +export const DEFAULT_OMP_DOCKER_IMAGE = + "node:24.15.0-bookworm-slim@sha256:4e6b70dd6cbfc88c8157ba19aa3d9f9cce6ba4703576d55459e45efcbc9c5f5d"; +export const DEFAULT_OMP_SANDBOX_PORT = 4000; +export const OMP_RUNTIME_MOUNT_PATH = "/opt/omp-eval"; +export const OMP_SANDBOX_WORKDIR = "/eval"; +export const OMP_SANDBOX_PROVIDER_ID = "omp-docker"; +export const OMP_SANDBOX_PNPM_VERSION = "10.28.2"; + +export class OmpDockerSandboxError extends Data.TaggedError("OmpDockerSandboxError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +export type OmpDockerFailure = OmpDockerSandboxError | HarnessSandboxAuthenticationError; + +const HELPER_PATH = `${OMP_SANDBOX_WORKDIR}/.omp-sandbox/helper.mjs`; +const PNPM_BIN = `${OMP_SANDBOX_WORKDIR}/.local/bin/pnpm`; +const BRIDGE_PORT_KEY = `${DEFAULT_OMP_SANDBOX_PORT}/tcp`; +const MEMORY_BYTES = 2 * 1024 * 1024 * 1024; +const NANO_CPUS = 2_000_000_000; +const PIDS_LIMIT = 512; +const EVAL_TMPFS_BYTES = 1_073_741_824; +const TMP_TMPFS_BYTES = 268_435_456; +const HOME_TMPFS_BYTES = 16_777_216; +const MAX_FILE_BYTES = 32 * 1024 * 1024; +const MAX_PROCESS_OUTPUT_BYTES = 8 * 1024 * 1024; +const CLEANUP_WAIT_MS = 8_000; +const CLEANUP_REQUEST_MS = 3_000; +const EXEC_EXIT_POLL_MS = 20; +const MAX_SPAWN_ACK_BYTES = 256; +const DOCKER_SOCKET_PATH = "/var/run/docker.sock"; +const SPAWN_READY_LINE = "OMP_SPAWN_READY"; +const LABEL_MANAGED = "askgina.evals.omp-sandbox"; +const TEXT_ENCODINGS = new Set(["utf-8", "utf8", "latin1", "utf16le"]); + +const CONTAINER_PATH = `${OMP_SANDBOX_WORKDIR}/.local/bin:${OMP_RUNTIME_MOUNT_PATH}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`; +const CONTAINER_ENV = [ + `HOME=${OMP_SANDBOX_WORKDIR}`, + "TMPDIR=/tmp", + `PATH=${CONTAINER_PATH}`, + `NPM_CONFIG_CACHE=${OMP_SANDBOX_WORKDIR}/.npm`, + `NPM_CONFIG_PREFIX=${OMP_SANDBOX_WORKDIR}/.local`, + "NPM_CONFIG_UPDATE_NOTIFIER=false", + "NPM_CONFIG_FUND=false", + `PNPM_HOME=${OMP_SANDBOX_WORKDIR}/.local/share/pnpm`, + `XDG_CACHE_HOME=${OMP_SANDBOX_WORKDIR}/.cache`, + `XDG_CONFIG_HOME=${OMP_SANDBOX_WORKDIR}/.config`, + `XDG_DATA_HOME=${OMP_SANDBOX_WORKDIR}/.local/share`, + `XDG_STATE_HOME=${OMP_SANDBOX_WORKDIR}/.local/state`, +] as const; + +const UnknownJsonString = Schema.fromJsonString(Schema.Unknown); +const encodeUnknownJson = Schema.encodeEffect(UnknownJsonString); +const decodeUnknownJson = Schema.decodeEffect(UnknownJsonString); +const HelperReadSchema = Schema.Struct({ + ok: Schema.Boolean, + exists: Schema.optional(Schema.Boolean), + contentBase64: Schema.optional(Schema.String), + error: Schema.optional(Schema.String), +}); +const HelperWriteSchema = Schema.Struct({ + ok: Schema.Boolean, + error: Schema.optional(Schema.String), +}); +const HelperReadJson = Schema.fromJsonString(HelperReadSchema); +const HelperWriteJson = Schema.fromJsonString(HelperWriteSchema); +const decodeHelperRead = Schema.decodeEffect(HelperReadJson); +const decodeHelperWrite = Schema.decodeEffect(HelperWriteJson); + +const HELPER_SOURCE = [ + "import { spawn } from 'node:child_process';", + "import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';", + "import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';", + "import { dirname, isAbsolute, join, normalize } from 'node:path';", + "import { stdin } from 'node:process';", + "", + "const WORK = '/eval';", + "const PROC_DIR = '/eval/.omp-procs';", + "const MAX_FILE_BYTES = 33554432;", + "const TOKEN = /^[a-f0-9-]{8,64}$/;", + "", + "const mode = process.argv[2] ?? 'rpc';", + "", + "const fail = () => {", + " process.stdout.write(JSON.stringify({ ok: false }));", + " process.exit(1);", + "};", + "", + "const readStdin = async () => {", + " const chunks = [];", + " for await (const chunk of stdin) chunks.push(chunk);", + " return Buffer.concat(chunks);", + "};", + "", + "const readJsonLine = async () => {", + " let buffer = '';", + " for await (const chunk of stdin) {", + " buffer += chunk.toString('utf8');", + " const index = buffer.indexOf('\\n');", + " if (index !== -1) {", + " stdin.pause();", + " return JSON.parse(buffer.slice(0, index));", + " }", + " }", + " return JSON.parse(buffer);", + "};", + "", + "const resolvePath = (value) => {", + " if (typeof value !== 'string' || value.length === 0 || value.includes('\\0')) {", + " throw new Error('path is invalid');", + " }", + " const resolved = normalize(isAbsolute(value) ? value : join(WORK, value));", + " if (resolved !== '/' && resolved.endsWith('/')) return resolved.slice(0, -1);", + " return resolved;", + "};", + "", + "const envRecord = (value) => {", + " const env = { ...process.env };", + " if (value == null || typeof value !== 'object' || Array.isArray(value)) return env;", + " for (const [key, entry] of Object.entries(value)) {", + " if (typeof key !== 'string' || key.length === 0 || key.includes('=') || key.includes('\\0')) continue;", + " if (typeof entry === 'string') env[key] = entry;", + " }", + " return env;", + "};", + "", + "const killPid = (pid) => {", + " for (const target of [-pid, pid]) {", + " try {", + " process.kill(target, 'SIGKILL');", + " } catch (error) {", + " if (!error || error.code !== 'ESRCH') throw error;", + " }", + " }", + "};", + "", + "const removeToken = (token) => {", + " try {", + " unlinkSync(`${PROC_DIR}/${token}`);", + " } catch (error) {", + " if (!error || error.code !== 'ENOENT') throw error;", + " }", + "};", + "", + "const runRpc = async () => {", + " const request = JSON.parse((await readStdin()).toString('utf8') || '{}');", + " if (request.op === 'read') {", + " const path = resolvePath(request.path);", + " try {", + " const info = await stat(path);", + " if (!info.isFile()) throw new Error('not a file');", + " if (info.size > MAX_FILE_BYTES) throw new Error('file exceeds byte limit');", + " const content = await readFile(path);", + " process.stdout.write(JSON.stringify({", + " ok: true,", + " exists: true,", + " contentBase64: content.toString('base64'),", + " }));", + " return;", + " } catch (error) {", + " if (error && error.code === 'ENOENT') {", + " process.stdout.write(JSON.stringify({ ok: true, exists: false }));", + " return;", + " }", + " throw error;", + " }", + " }", + " if (request.op === 'write') {", + " const path = resolvePath(request.path);", + " if (typeof request.contentBase64 !== 'string') throw new Error('content is invalid');", + " const content = Buffer.from(request.contentBase64, 'base64');", + " if (content.byteLength > MAX_FILE_BYTES) throw new Error('file exceeds byte limit');", + " await mkdir(dirname(path), { recursive: true });", + " await writeFile(path, content);", + " process.stdout.write(JSON.stringify({ ok: true }));", + " return;", + " }", + " throw new Error('unknown helper operation');", + "};", + "", + "const runSpawn = async () => {", + " const request = await readJsonLine();", + " if (typeof request.command !== 'string' || request.command.length === 0) {", + " throw new Error('command is invalid');", + " }", + " if (typeof request.token !== 'string' || !TOKEN.test(request.token)) {", + " throw new Error('token is invalid');", + " }", + " const cwd = request.cwd == null ? WORK : resolvePath(request.cwd);", + " mkdirSync(PROC_DIR, { recursive: true, mode: 0o700 });", + " const cancelled = `${PROC_DIR}/${request.token}.cancelled`;", + " if (existsSync(cancelled)) { process.exitCode = 1; return; }", + " const child = spawn('/bin/sh', ['-c', request.command], {", + " cwd,", + " env: envRecord(request.env),", + " stdio: ['ignore', 'pipe', 'pipe'],", + " detached: true,", + " });", + " if (child.pid == null) throw new Error('process did not start');", + " writeFileSync(`${PROC_DIR}/${request.token}`, String(child.pid), { encoding: 'utf8', mode: 0o600 });", + " if (existsSync(cancelled)) killPid(child.pid);", + ` process.stdout.write('${SPAWN_READY_LINE}\\n');`, + " const shutdown = () => {", + " killPid(child.pid);", + " };", + " process.on('SIGHUP', shutdown);", + " process.on('SIGTERM', shutdown);", + " process.on('SIGINT', shutdown);", + " child.stdout.pipe(process.stdout);", + " child.stderr.pipe(process.stderr);", + " child.once('close', (code) => {", + " try {", + " removeToken(request.token);", + " process.exitCode = code ?? 1;", + " } catch {", + " process.exitCode = 1;", + " }", + " });", + "};", + "", + "const runKill = async () => {", + " const request = JSON.parse((await readStdin()).toString('utf8') || '{}');", + " if (typeof request.token !== 'string' || !TOKEN.test(request.token)) {", + " throw new Error('token is invalid');", + " }", + " mkdirSync(PROC_DIR, { recursive: true, mode: 0o700 });", + " writeFileSync(`${PROC_DIR}/${request.token}.cancelled`, '', { mode: 0o600 });", + " let pid = 0;", + " try {", + " pid = Number.parseInt(readFileSync(`${PROC_DIR}/${request.token}`, 'utf8'), 10);", + " } catch (error) {", + " if (error && error.code === 'ENOENT') {", + " process.stdout.write(JSON.stringify({ ok: true }));", + " return;", + " }", + " throw error;", + " }", + " if (!Number.isInteger(pid) || pid <= 1) throw new Error('stored process is invalid');", + " killPid(pid);", + " removeToken(request.token);", + " process.stdout.write(JSON.stringify({ ok: true }));", + "};", + "", + "const main = async () => {", + " if (mode === 'spawn') return runSpawn();", + " if (mode === 'kill') return runKill();", + " return runRpc();", + "};", + "", + "main().catch(fail);", +].join("\n"); + +export interface OmpDockerSandboxOptions { + readonly runtimeDirectory: string; + readonly image?: string; + readonly network?: string; +} + +type SandboxSession = ReturnType; +type DockerClient = InstanceType; +type DockerContainer = import("dockerode").Container; +type DockerExec = import("dockerode").Exec; +type DockerNetwork = import("dockerode").Network; + +interface SpawnHandle { + readonly token: string; + readonly exec: DockerExec; + readonly stream: Duplex; + readonly stdout: PassThrough; + readonly stderr: PassThrough; + killed: boolean; + abortError?: OmpDockerSandboxError; + abortTeardown?: () => void; +} + +interface StartedExec { + readonly exec: DockerExec; + readonly stream: Duplex; + readonly stdout: PassThrough; + readonly stderr: PassThrough; +} + +interface ValidatedSandboxOptions { + readonly runtimeDirectory: string; + readonly image: string; + readonly callerNetwork: string | undefined; +} + +const ignoreError = (_error: unknown): void => undefined; + +const sandboxError = (message: string, cause?: unknown): OmpDockerSandboxError => + new OmpDockerSandboxError({ + message, + ...(cause === undefined ? {} : { cause }), + }); + +const abortError = (signal: AbortSignal): OmpDockerSandboxError => + sandboxError("OMP sandbox was aborted.", signal.reason); + +const dockerCause = (error: unknown): unknown => + error instanceof OmpDockerSandboxError && error.cause !== undefined ? error.cause : error; + +const dockerStatus = (error: unknown): number | undefined => { + const candidate = dockerCause(error); + if (typeof candidate !== "object" || candidate === null) return undefined; + if ("statusCode" in candidate && typeof candidate.statusCode === "number") + return candidate.statusCode; + if ("status" in candidate && typeof candidate.status === "number") return candidate.status; + return undefined; +}; + +const isIgnorableDockerError = (error: unknown): boolean => { + const status = dockerStatus(error); + if (status === 304 || status === 404) return true; + const candidate = dockerCause(error); + const message = candidate instanceof Error ? candidate.message : String(candidate); + return /no such (container|network|exec)|is not running|already (stopped|paused)|not found/iu.test( + message, + ); +}; + +const isDockerAuthError = (error: unknown): boolean => { + const status = dockerStatus(error); + return status === 401 || status === 403; +}; + +const requireAbsolutePath = (value: string, field: string): string => { + if ( + typeof value !== "string" || + value.length === 0 || + value.includes("\0") || + /\s/u.test(value) + ) { + throw sandboxError(`OMP Docker ${field} is invalid.`); + } + if (value.startsWith("/") === false) { + throw sandboxError(`OMP Docker ${field} must be an absolute path.`); + } + return value; +}; + +const resolveImage = (image: string | undefined): string => { + if (image === undefined || image.length === 0) return DEFAULT_OMP_DOCKER_IMAGE; + if (image.length > 256 || /[\s\0]/u.test(image)) { + throw sandboxError("OMP Docker image override is invalid."); + } + if (image.includes("@") && /@sha256:[a-fA-F0-9]{64}$/u.test(image) === false) { + throw sandboxError("OMP Docker image override must be digest-pinned."); + } + return image; +}; + +const requireNetworkName = (network: string): string => { + if (/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(network) === false) { + throw sandboxError("OMP Docker network option is invalid."); + } + return network; +}; + +const encodeText = ( + content: string, + encoding: string | undefined, +): Effect.Effect => { + const label = encoding ?? "utf-8"; + if (TEXT_ENCODINGS.has(label) === false) { + return Effect.fail(sandboxError(`OMP sandbox text encoding is unsupported: ${label}`)); + } + return Effect.succeed( + Buffer.from(content, label === "utf-8" ? "utf8" : (label as BufferEncoding)), + ); +}; + +const decodeText = ( + bytes: Uint8Array, + encoding: string | undefined, +): Effect.Effect => { + const label = encoding ?? "utf-8"; + if (TEXT_ENCODINGS.has(label) === false) { + return Effect.fail(sandboxError(`OMP sandbox text encoding is unsupported: ${label}`)); + } + return Effect.succeed( + Buffer.from(bytes).toString(label === "utf-8" ? "utf8" : (label as BufferEncoding)), + ); +}; + +const sliceTextLines = (text: string, startLine?: number, endLine?: number): string => { + const lines = text.split("\n"); + const start = Math.max(1, startLine ?? 1); + const end = Math.max(start - 1, endLine ?? lines.length); + return lines.slice(start - 1, end).join("\n"); +}; + +const toWebStream = (stream: PassThrough): ReadableStream => + Duplex.toWeb(stream).readable as ReadableStream; + +const abortEffect = (signal: AbortSignal): Effect.Effect => + Effect.callback((resume) => { + const fail = () => { + resume(Effect.fail(abortError(signal))); + }; + if (signal.aborted === true) { + fail(); + return; + } + signal.addEventListener("abort", fail, { once: true }); + return Effect.sync(() => { + signal.removeEventListener("abort", fail); + }); + }); + +const withAbort = ( + effect: Effect.Effect, + signal: AbortSignal | undefined, +): Effect.Effect => + signal === undefined ? effect : Effect.raceFirst(effect, abortEffect(signal)); + +const runBoundary = (effect: Effect.Effect, signal?: AbortSignal): Promise => + Effect.runPromise(withAbort(effect, signal)); + +const fromDocker = ( + start: (signal: AbortSignal) => Promise, + onLate: (value: A) => void = ignoreError, +): Effect.Effect => + Effect.callback((resume, signal) => { + let settled = false; + const work = start(signal); + const settleFail = (error: unknown) => { + if (settled === true) return; + settled = true; + resume(Effect.fail(sandboxError("OMP Docker operation failed.", error))); + }; + const settleOk = (value: A) => { + if (settled === true || signal.aborted === true) { + onLate(value); + return; + } + settled = true; + resume(Effect.succeed(value)); + }; + void work.then(settleOk, settleFail); + if (signal.aborted === true) { + settled = true; + resume(Effect.fail(abortError(signal))); + } + }); + +const collectPassThrough = ( + stream: PassThrough, + maxBytes: number, +): Effect.Effect => + Effect.callback((resume) => { + const chunks: Buffer[] = []; + let size = 0; + let settled = false; + const finish = (next: Effect.Effect) => { + if (settled === true) return; + settled = true; + resume(next); + }; + const onData = (chunk: Buffer) => { + if (settled === true) return; + if (size + chunk.byteLength > maxBytes) { + finish(Effect.fail(sandboxError("OMP sandbox stream exceeded byte limit."))); + return; + } + chunks.push(Buffer.from(chunk)); + size += chunk.byteLength; + }; + const onError = () => { + finish(Effect.fail(sandboxError("OMP sandbox stream failed."))); + }; + const onEnd = () => { + finish(Effect.succeed(Buffer.concat(chunks, size))); + }; + stream.on("data", onData); + stream.once("error", onError); + stream.once("end", onEnd); + return Effect.sync(() => { + stream.off("data", onData); + stream.off("error", onError); + stream.off("end", onEnd); + }); + }); + +const waitForSpawnReady = (stream: PassThrough): Effect.Effect => + Effect.callback((resume) => { + let buffer: Buffer = Buffer.alloc(0); + let settled = false; + const finish = (next: Effect.Effect) => { + if (settled === true) return; + settled = true; + stream.off("data", onData); + stream.off("error", onError); + stream.off("end", onEnd); + resume(next); + }; + const onData = (chunk: Buffer) => { + if (settled === true) return; + buffer = buffer.byteLength === 0 ? chunk : Buffer.concat([buffer, chunk]); + const newline = buffer.indexOf(0x0a); + if (newline === -1) { + if (buffer.byteLength > MAX_SPAWN_ACK_BYTES) { + finish(Effect.fail(sandboxError("OMP sandbox process did not start."))); + } + return; + } + const line = buffer.subarray(0, newline).toString("utf8"); + const rest = buffer.subarray(newline + 1); + if (newline > MAX_SPAWN_ACK_BYTES || line !== SPAWN_READY_LINE) { + finish(Effect.fail(sandboxError("OMP sandbox process did not start."))); + return; + } + settled = true; + stream.off("data", onData); + stream.off("error", onError); + stream.off("end", onEnd); + stream.pause(); + if (rest.byteLength > 0) stream.unshift(rest); + resume(Effect.void); + }; + const onError = () => { + finish(Effect.fail(sandboxError("OMP sandbox process did not start."))); + }; + const onEnd = () => { + finish(Effect.fail(sandboxError("OMP sandbox process did not start."))); + }; + stream.on("data", onData); + stream.once("error", onError); + stream.once("end", onEnd); + return Effect.sync(() => { + if (settled === true) return; + stream.off("data", onData); + stream.off("error", onError); + stream.off("end", onEnd); + }); + }); + +const readStreamBytes = ( + stream: ReadableStream, + maxBytes: number, +): Effect.Effect => + Effect.callback((resume) => { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + const pump = (): void => { + void reader.read().then( + (result) => { + if (result.done === true) { + const bytes = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + resume(Effect.succeed(bytes)); + return; + } + const value = result.value; + if (value === undefined) { + pump(); + return; + } + if (size + value.byteLength > maxBytes) { + resume(Effect.fail(sandboxError("OMP sandbox write exceeds byte limit."))); + return; + } + chunks.push(value); + size += value.byteLength; + pump(); + }, + (error) => { + resume(Effect.fail(sandboxError("OMP sandbox read failed.", error))); + }, + ); + }; + pump(); + return Effect.sync(() => { + void reader.cancel(); + }); + }); + +const collectUtf8 = ( + stream: ReadableStream, + maxBytes: number, +): Effect.Effect => + readStreamBytes(stream, maxBytes).pipe( + Effect.map((bytes) => Buffer.from(bytes).toString("utf8")), + ); + +const followPull = ( + docker: DockerClient, + stream: NodeJS.ReadableStream, +): Effect.Effect => + Effect.callback((resume) => { + docker.modem.followProgress(stream, (error) => { + if (error !== null && error !== undefined) { + resume(Effect.fail(sandboxError("OMP Docker image pull failed.", error))); + return; + } + resume(Effect.void); + }); + }); + +const authFailure = ( + message: string, + cause: unknown, +): Effect.Effect => + Effect.fail( + new HarnessSandboxAuthenticationError({ + message, + sandboxProviderId: OMP_SANDBOX_PROVIDER_ID, + cause, + }), + ); + +const ensureImage = (docker: DockerClient, image: string): Effect.Effect => + fromDocker(() => docker.getImage(image).inspect()).pipe( + Effect.asVoid, + Effect.matchEffect({ + onSuccess: (): Effect.Effect => Effect.void, + onFailure: (error): Effect.Effect => { + if (isDockerAuthError(error) === true) { + return authFailure("OMP Docker image inspect was not authorized.", error); + } + if (dockerStatus(error) !== 404 && isIgnorableDockerError(error) === false) { + return sandboxError("OMP Docker image inspect failed.", error); + } + return fromDocker((signal) => docker.pull(image, { abortSignal: signal })).pipe( + Effect.flatMap((pullStream) => followPull(docker, pullStream)), + Effect.matchEffect({ + onSuccess: (): Effect.Effect => Effect.void, + onFailure: (pullError): Effect.Effect => { + if (isDockerAuthError(pullError) === true) { + return authFailure("OMP Docker image pull was not authorized.", pullError); + } + const named = image.includes("@") ? image.slice(0, image.indexOf("@")) : image; + return sandboxError(`OMP sandbox failed to pull ${named}.`, pullError); + }, + }), + ); + }, + }), + ); + +const waitForExecExit = (exec: DockerExec): Effect.Effect => + Effect.gen(function* () { + for (;;) { + const info = yield* fromDocker(() => exec.inspect()); + if (info.Running === false) return info.ExitCode ?? 1; + yield* Effect.sleep(Duration.millis(EXEC_EXIT_POLL_MS)); + } + }); + +const collectStarted = ( + started: StartedExec, + stdoutLimit: number, + stderrLimit: number, +): Effect.Effect<{ stdout: Buffer; stderr: Buffer; exitCode: number }, OmpDockerSandboxError> => + Effect.all( + [ + collectPassThrough(started.stdout, stdoutLimit), + collectPassThrough(started.stderr, stderrLimit), + waitForExecExit(started.exec), + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.map(([stdout, stderr, exitCode]) => ({ stdout, stderr, exitCode })), + Effect.ensuring( + Effect.sync(() => { + started.stream.destroy(); + }), + ), + ); + +const attemptCleanup = ( + effect: Effect.Effect, +): Effect.Effect => + effect.pipe( + Effect.match({ + onSuccess: () => true, + onFailure: (error) => isIgnorableDockerError(error), + }), + ); + +const attachDemux = ( + docker: DockerClient, + stream: Duplex, +): { readonly stdout: PassThrough; readonly stderr: PassThrough } => { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + docker.modem.demuxStream(stream, stdout, stderr); + const finish = () => { + stdout.end(); + stderr.end(); + }; + stream.once("end", finish); + stream.once("close", finish); + stream.once("error", (error: Error) => { + stdout.destroy(error); + stderr.destroy(error); + }); + return { stdout, stderr }; +}; + +const forceRemoveContainer = ( + container: DockerContainer, +): Effect.Effect => + Effect.gen(function* () { + const removed = yield* attemptCleanup( + fromDocker((signal) => + container.remove({ + force: true, + v: true, + abortSignal: AbortSignal.any([signal, AbortSignal.timeout(CLEANUP_REQUEST_MS)]), + }), + ).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(CLEANUP_REQUEST_MS), + orElse: () => Effect.fail(sandboxError("OMP sandbox cleanup failed.")), + }), + ), + ); + if (removed === false) { + return yield* sandboxError("OMP sandbox cleanup failed."); + } + }); + +const removeOwnedNetwork = (network: DockerNetwork): Effect.Effect => + Effect.gen(function* () { + const removed = yield* attemptCleanup( + fromDocker((signal) => + network.remove({ + abortSignal: AbortSignal.any([signal, AbortSignal.timeout(CLEANUP_REQUEST_MS)]), + }), + ).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(CLEANUP_REQUEST_MS), + orElse: () => Effect.fail(sandboxError("OMP sandbox cleanup failed.")), + }), + ), + ); + if (removed === false) { + return yield* sandboxError("OMP sandbox cleanup failed."); + } + }); + +class OwnedSessionResources { + container: DockerContainer | undefined; + ownedNetwork: DockerNetwork | undefined; + lost = false; + private readonly spawns = new Set(); + private inFlight: Promise | undefined; + private finished = false; + private dirty = false; + + markLost(): void { + this.lost = true; + } + + claimContainer(container: DockerContainer): void { + this.container ??= container; + this.dirty = true; + if (this.lost === true) this.scheduleLateCleanup(); + } + + claimOwnedNetwork(network: DockerNetwork): void { + this.ownedNetwork ??= network; + this.dirty = true; + if (this.lost === true) this.scheduleLateCleanup(); + } + + trackSpawn(handle: SpawnHandle): void { + this.spawns.add(handle); + } + + forgetSpawn(handle: SpawnHandle): void { + this.spawns.delete(handle); + } + + dispose(): Effect.Effect { + this.lost = true; + return Effect.tryPromise({ + try: () => this.beginCleanup(), + catch: (error) => + error instanceof OmpDockerSandboxError + ? error + : sandboxError("OMP sandbox cleanup failed."), + }); + } + + disposeBounded(): Effect.Effect { + this.lost = true; + return Effect.race( + this.dispose().pipe(Effect.ignore), + Effect.sleep(Duration.millis(CLEANUP_WAIT_MS)), + ).pipe( + Effect.tap(() => + Effect.sync(() => { + void this.beginCleanup().catch(ignoreError); + }), + ), + Effect.asVoid, + ); + } + + private scheduleLateCleanup(): void { + this.lost = true; + this.finished = false; + this.dirty = true; + void this.beginCleanup().catch(ignoreError); + } + + private beginCleanup(): Promise { + if (this.finished === true && this.dirty === false && this.inFlight === undefined) { + return Promise.resolve(); + } + if (this.inFlight !== undefined) return this.inFlight; + this.finished = false; + const work = Effect.runPromise(this.cleanupKnown()).then( + () => { + if (this.inFlight !== work) return; + this.inFlight = undefined; + if (this.dirty === true) return this.beginCleanup(); + this.finished = true; + }, + (error: unknown) => { + if (this.inFlight === work) this.inFlight = undefined; + throw error; + }, + ); + this.inFlight = work; + return work; + } + + private cleanupKnown(): Effect.Effect { + const self = this; + return Effect.gen(function* () { + self.dirty = false; + const container = self.container; + const ownedNetwork = self.ownedNetwork; + let failed = false; + for (const handle of self.spawns) { + handle.abortTeardown?.(); + handle.abortTeardown = undefined; + handle.stream.destroy(); + handle.killed = true; + self.spawns.delete(handle); + } + if (container !== undefined) { + const removed = yield* attemptCleanup(forceRemoveContainer(container)); + if (removed === false) failed = true; + } + if (ownedNetwork !== undefined) { + const removed = yield* attemptCleanup(removeOwnedNetwork(ownedNetwork)); + if (removed === false) failed = true; + } + if (failed === true) { + return yield* sandboxError("OMP sandbox cleanup failed."); + } + }); + } +} + +const startExec = ( + docker: DockerClient, + container: DockerContainer, + command: readonly string[], + stdin: Uint8Array, +): Effect.Effect => + Effect.gen(function* () { + const exec = yield* fromDocker((signal) => + container.exec({ + Cmd: [...command], + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Tty: false, + abortSignal: signal, + }), + ); + const stream = yield* fromDocker((signal) => + exec.start({ + hijack: true, + stdin: true, + Tty: false, + abortSignal: signal, + }), + ); + const { stdout, stderr } = attachDemux(docker, stream); + stream.write(stdin); + stream.end(); + return { exec, stream, stdout, stderr }; + }); + +const helperRpc = ( + docker: DockerClient, + container: DockerContainer, + request: Record, +): Effect.Effect => + Effect.gen(function* () { + const body = yield* encodeUnknownJson(request).pipe( + Effect.mapError((error) => sandboxError("OMP sandbox helper request is invalid.", error)), + ); + const started = yield* startExec( + docker, + container, + ["node", HELPER_PATH], + Buffer.from(body, "utf8"), + ); + const collected = yield* collectStarted(started, MAX_FILE_BYTES * 2, 65_536); + if (collected.exitCode !== 0) { + return yield* sandboxError("OMP sandbox helper failed."); + } + return yield* decodeUnknownJson(collected.stdout.toString("utf8")).pipe( + Effect.mapError(() => sandboxError("OMP sandbox helper returned invalid JSON.")), + ); + }); + +const killSpawn = ( + docker: DockerClient, + container: DockerContainer, + token: string, +): Effect.Effect => + Effect.gen(function* () { + const body = yield* encodeUnknownJson({ token }).pipe( + Effect.mapError(() => sandboxError("OMP sandbox kill request is invalid.")), + ); + const started = yield* startExec( + docker, + container, + ["node", HELPER_PATH, "kill"], + Buffer.from(body, "utf8"), + ); + const collected = yield* collectStarted(started, 4096, 4096); + if (collected.exitCode !== 0) { + return yield* sandboxError("OMP sandbox process kill failed."); + } + const result = yield* decodeHelperWrite(collected.stdout.toString("utf8")).pipe( + Effect.mapError(() => sandboxError("OMP sandbox process kill failed.")), + ); + if (result.ok !== true) { + return yield* sandboxError("OMP sandbox process kill failed."); + } + }); + +const installHelper = ( + docker: DockerClient, + container: DockerContainer, +): Effect.Effect => + Effect.gen(function* () { + const started = yield* startExec( + docker, + container, + [ + "node", + "-e", + "const fs = require('node:fs'); fs.mkdirSync('/eval/.omp-sandbox', { mode: 0o700 }); fs.writeFileSync(process.argv[1], fs.readFileSync(0), { flag: 'wx', mode: 0o500 });", + HELPER_PATH, + ], + Buffer.from(HELPER_SOURCE, "utf8"), + ); + const result = yield* collectStarted(started, 4096, 4096); + if (result.exitCode !== 0) { + return yield* sandboxError("OMP sandbox helper installation failed."); + } + }); + +const provisionPnpm = ( + docker: DockerClient, + container: DockerContainer, +): Effect.Effect => + Effect.gen(function* () { + const install = yield* startExec( + docker, + container, + [ + "npm", + "install", + "-g", + `pnpm@${OMP_SANDBOX_PNPM_VERSION}`, + "--prefix", + `${OMP_SANDBOX_WORKDIR}/.local`, + "--omit=dev", + "--no-audit", + "--no-fund", + ], + Buffer.alloc(0), + ); + const installed = yield* collectStarted(install, 65_536, 65_536); + if (installed.exitCode !== 0) { + return yield* sandboxError( + `OMP sandbox failed to provision pnpm@${OMP_SANDBOX_PNPM_VERSION}.`, + ); + } + const verify = yield* startExec(docker, container, [PNPM_BIN, "--version"], Buffer.alloc(0)); + const verified = yield* collectStarted(verify, 4096, 4096); + const version = verified.stdout.toString("utf8").trim(); + if (verified.exitCode !== 0 || version !== OMP_SANDBOX_PNPM_VERSION) { + return yield* sandboxError(`OMP sandbox pnpm is not ${OMP_SANDBOX_PNPM_VERSION}.`); + } + }); + +const inspectPublishedPort = ( + container: DockerContainer, + port: number, + protocol: "http" | "https" | "ws" | undefined, +): Effect.Effect => + fromDocker(() => container.inspect()).pipe( + Effect.flatMap((info) => { + if (port !== DEFAULT_OMP_SANDBOX_PORT) { + return sandboxError(`OMP sandbox port ${port} is not published.`); + } + const binding = info.NetworkSettings.Ports?.[BRIDGE_PORT_KEY]?.[0]; + const hostPort = binding?.HostPort; + const hostIp = binding?.HostIp; + if (hostPort === undefined || hostPort.length === 0) { + return sandboxError("OMP sandbox bridge port is not published on loopback."); + } + if (hostIp !== undefined && hostIp.length > 0 && hostIp !== "127.0.0.1") { + return sandboxError("OMP sandbox bridge port is not bound to loopback."); + } + const scheme = protocol ?? "http"; + return Effect.succeed({ url: `${scheme}://127.0.0.1:${hostPort}` }); + }), + ); + +const createOwnedNetwork = ( + docker: DockerClient, + resources: OwnedSessionResources, +): Effect.Effect => { + const networkName = `omp-eval-net-${randomBytes(8).toString("hex")}`; + return fromDocker( + (signal) => + docker.createNetwork({ + Name: networkName, + Driver: "bridge", + CheckDuplicate: true, + Internal: false, + Attachable: false, + EnableIPv6: false, + Labels: { + [LABEL_MANAGED]: "1", + [`${LABEL_MANAGED}.role`]: "session", + }, + abortSignal: signal, + }), + (network) => { + resources.claimOwnedNetwork(network); + }, + ).pipe( + Effect.map((network) => { + resources.claimOwnedNetwork(network); + return network.id; + }), + ); +}; + +const createOwnedContainer = ( + docker: DockerClient, + resources: OwnedSessionResources, + options: ValidatedSandboxOptions, + networkMode: string, + sessionId: string | undefined, +): Effect.Effect => { + const containerName = `omp-eval-${randomBytes(8).toString("hex")}`; + return fromDocker( + (signal) => + docker.createContainer({ + name: containerName, + Image: options.image, + User: "node", + WorkingDir: OMP_SANDBOX_WORKDIR, + Cmd: ["node", "-e", "setInterval(()=>{}, 1<<30)"], + Env: [...CONTAINER_ENV], + ExposedPorts: { [BRIDGE_PORT_KEY]: {} }, + Labels: { + [LABEL_MANAGED]: "1", + [`${LABEL_MANAGED}.image`]: options.image.includes("@") + ? options.image.slice(0, options.image.indexOf("@")) + : options.image, + ...(sessionId === undefined + ? {} + : { [`${LABEL_MANAGED}.session`]: sessionId.slice(0, 64) }), + }, + HostConfig: { + AutoRemove: false, + NetworkMode: networkMode, + PortBindings: { + [BRIDGE_PORT_KEY]: [{ HostIp: "127.0.0.1", HostPort: "0" }], + }, + Mounts: [ + { + Target: OMP_RUNTIME_MOUNT_PATH, + Source: options.runtimeDirectory, + Type: "bind", + ReadOnly: true, + BindOptions: { Propagation: "rprivate" }, + }, + ], + Tmpfs: { + [OMP_SANDBOX_WORKDIR]: `rw,exec,nosuid,nodev,mode=1777,size=${EVAL_TMPFS_BYTES}`, + "/tmp": `rw,exec,nosuid,nodev,mode=1777,size=${TMP_TMPFS_BYTES}`, + "/home/node": `rw,nosuid,nodev,mode=1777,size=${HOME_TMPFS_BYTES}`, + }, + ReadonlyRootfs: true, + CapDrop: ["ALL"], + SecurityOpt: ["no-new-privileges:true"], + Privileged: false, + PublishAllPorts: false, + Init: true, + Memory: MEMORY_BYTES, + MemorySwap: MEMORY_BYTES, + NanoCpus: NANO_CPUS, + PidsLimit: PIDS_LIMIT, + RestartPolicy: { Name: "no" }, + ExtraHosts: [], + LogConfig: { + Type: "json-file", + Config: { "max-size": "1m", "max-file": "1" }, + }, + }, + abortSignal: signal, + }), + (container) => { + resources.claimContainer(container); + }, + ).pipe( + Effect.map((container) => { + resources.claimContainer(container); + return container; + }), + ); +}; + +const createSessionEffect = ( + options: ValidatedSandboxOptions, + sessionOptions: + | { + sessionId?: string; + abortSignal?: AbortSignal; + identity?: string; + onFirstCreate?: ( + session: SandboxSession, + opts: { abortSignal?: AbortSignal }, + ) => Promise; + } + | undefined, +): Effect.Effect => { + const docker = new Dockerode({ socketPath: DOCKER_SOCKET_PATH }); + const resources = new OwnedSessionResources(); + let published = false; + + return Effect.gen(function* () { + yield* ensureImage(docker, options.image); + + let networkMode = "bridge"; + if (options.callerNetwork !== undefined) { + const callerNetwork = options.callerNetwork; + yield* fromDocker(() => docker.getNetwork(callerNetwork).inspect()).pipe( + Effect.matchEffect({ + onSuccess: (): Effect.Effect => Effect.void, + onFailure: (error): Effect.Effect => + isDockerAuthError(error) === true + ? authFailure("OMP Docker network inspect was not authorized.", error) + : sandboxError("OMP Docker caller network is not available.", error), + }), + ); + networkMode = callerNetwork; + } else { + networkMode = yield* createOwnedNetwork(docker, resources); + } + + const createdContainer = yield* createOwnedContainer( + docker, + resources, + options, + networkMode, + sessionOptions?.sessionId, + ); + yield* fromDocker((signal) => createdContainer.start({ abortSignal: signal })); + yield* installHelper(docker, createdContainer); + yield* provisionPnpm(docker, createdContainer); + const session = createSessionSurface({ docker, container: createdContainer, resources }); + if (sessionOptions?.onFirstCreate !== undefined) { + const onFirstCreate = sessionOptions.onFirstCreate; + yield* fromDocker((signal) => + Promise.resolve(onFirstCreate(session.restricted(), { abortSignal: signal })), + ); + } + return session; + }).pipe( + Effect.tap(() => + Effect.sync(() => { + published = true; + }), + ), + Effect.ensuring( + Effect.suspend(() => { + if (published === true) return Effect.void; + resources.markLost(); + return resources.disposeBounded(); + }), + ), + ); +}; + +/** + * Synchronous Docker sandbox provider for official HarnessAgent/createACP. + * Construction performs no Docker or filesystem I/O. + * + * `runtimeDirectory` is bind-mounted read-only at `/opt/omp-eval`. + * The provider is non-resumable: `stop` and `destroy` are the same + * idempotent dispose (container stop+force-remove with volumes, then + * owned network only). HarnessAgent `cleanupAfterStartFailure` calls + * only `stop()`, so that path must not leave a stopped container. + */ +export function createOmpDockerSandbox(options: OmpDockerSandboxOptions): HarnessV1SandboxProvider { + const validated: ValidatedSandboxOptions = { + runtimeDirectory: requireAbsolutePath(options.runtimeDirectory, "runtimeDirectory"), + image: resolveImage(options.image), + callerNetwork: options.network === undefined ? undefined : requireNetworkName(options.network), + }; + + return { + specificationVersion: "harness-sandbox-v1", + providerId: OMP_SANDBOX_PROVIDER_ID, + createSession: (sessionOptions) => + runBoundary(createSessionEffect(validated, sessionOptions), sessionOptions?.abortSignal), + }; +} + +const createSessionSurface = ({ + docker, + container, + resources, +}: { + readonly docker: DockerClient; + readonly container: DockerContainer; + readonly resources: OwnedSessionResources; +}): HarnessV1NetworkSandboxSession => { + let closed = false; + + const assertOpen = (): Effect.Effect => + closed === true ? Effect.fail(sandboxError("OMP sandbox session is closed.")) : Effect.void; + + const close = (): Promise => { + closed = true; + return Effect.runPromise(resources.dispose()); + }; + + const readBinary = (path: string): Effect.Effect => + Effect.gen(function* () { + yield* assertOpen(); + const raw = yield* helperRpc(docker, container, { op: "read", path }); + const encoded = yield* encodeUnknownJson(raw).pipe( + Effect.mapError((error) => sandboxError("OMP sandbox read is invalid.", error)), + ); + const result = yield* decodeHelperRead(encoded).pipe( + Effect.mapError((error) => sandboxError("OMP sandbox read is invalid.", error)), + ); + if (result.ok === false) { + return yield* sandboxError("OMP sandbox read failed."); + } + if (result.exists !== true) return null; + if (result.contentBase64 === undefined) { + return yield* sandboxError("OMP sandbox read returned no content."); + } + return Buffer.from(result.contentBase64, "base64"); + }); + + const writeBinary = ( + path: string, + content: Uint8Array, + ): Effect.Effect => + Effect.gen(function* () { + yield* assertOpen(); + if (content.byteLength > MAX_FILE_BYTES) { + return yield* sandboxError("OMP sandbox write exceeds byte limit."); + } + const raw = yield* helperRpc(docker, container, { + op: "write", + path, + contentBase64: Buffer.from(content).toString("base64"), + }); + const encoded = yield* encodeUnknownJson(raw).pipe( + Effect.mapError((error) => sandboxError("OMP sandbox write is invalid.", error)), + ); + const result = yield* decodeHelperWrite(encoded).pipe( + Effect.mapError((error) => sandboxError("OMP sandbox write is invalid.", error)), + ); + if (result.ok === false) { + return yield* sandboxError("OMP sandbox write failed."); + } + }); + + const teardownAbort = (handle: SpawnHandle): void => { + handle.abortTeardown?.(); + handle.abortTeardown = undefined; + }; + + const killHandle = (handle: SpawnHandle): Effect.Effect => + Effect.suspend(() => { + if (handle.killed === true) return Effect.void; + handle.killed = true; + teardownAbort(handle); + return killSpawn(docker, container, handle.token).pipe( + Effect.ensuring( + Effect.sync(() => { + handle.stream.destroy(); + resources.forgetSpawn(handle); + }), + ), + Effect.tapError(() => + Effect.sync(() => { + handle.killed = false; + }), + ), + ); + }); + + const bindAbort = (handle: SpawnHandle, signal: AbortSignal | undefined): void => { + if (signal === undefined) return; + const onAbort = (): void => { + handle.abortError = abortError(signal); + void Effect.runPromise(killHandle(handle)).catch(ignoreError); + }; + if (signal.aborted === true) { + onAbort(); + return; + } + signal.addEventListener("abort", onAbort, { once: true }); + handle.abortTeardown = () => { + signal.removeEventListener("abort", onAbort); + }; + }; + + const startSpawn = (options: { + readonly command: string; + readonly workingDirectory?: string; + readonly env?: Record; + readonly abortSignal?: AbortSignal; + }): Effect.Effect => + Effect.gen(function* () { + yield* assertOpen(); + if (typeof options.command !== "string" || options.command.length === 0) { + return yield* sandboxError("OMP sandbox command is invalid."); + } + const token = randomBytes(16).toString("hex"); + const body = yield* encodeUnknownJson({ + token, + command: options.command, + cwd: options.workingDirectory, + env: options.env, + }).pipe(Effect.mapError(() => sandboxError("OMP sandbox spawn request is invalid."))); + const started = yield* startExec( + docker, + container, + ["node", HELPER_PATH, "spawn"], + Buffer.from(`${body}\n`, "utf8"), + ); + const handle: SpawnHandle = { + token, + exec: started.exec, + stream: started.stream, + stdout: started.stdout, + stderr: started.stderr, + killed: false, + }; + resources.trackSpawn(handle); + yield* waitForSpawnReady(handle.stdout).pipe( + Effect.matchEffect({ + onSuccess: () => Effect.void, + onFailure: (error) => + Effect.ignore(killHandle(handle)).pipe(Effect.andThen(Effect.fail(error))), + }), + Effect.onInterrupt(() => Effect.ignore(killHandle(handle))), + ); + if (options.abortSignal?.aborted === true) { + yield* killHandle(handle); + return yield* abortError(options.abortSignal); + } + bindAbort(handle, options.abortSignal); + return handle; + }); + + const waitHandle = ( + handle: SpawnHandle, + ): Effect.Effect<{ readonly exitCode: number }, OmpDockerSandboxError> => + Effect.suspend(() => + handle.abortError !== undefined + ? Effect.fail(handle.abortError) + : waitForExecExit(handle.exec).pipe(Effect.map((exitCode) => ({ exitCode }))), + ).pipe( + Effect.ensuring( + Effect.sync(() => { + teardownAbort(handle); + resources.forgetSpawn(handle); + }), + ), + ); + + const spawnProcess = (options: { + readonly command: string; + readonly workingDirectory?: string; + readonly env?: Record; + readonly abortSignal?: AbortSignal; + }) => + runBoundary(startSpawn(options), options.abortSignal).then((handle) => ({ + stdout: toWebStream(handle.stdout), + stderr: toWebStream(handle.stderr), + wait: () => runBoundary(waitHandle(handle), options.abortSignal), + kill: () => Effect.runPromise(killHandle(handle)), + })); + + const runCommand = (options: { + readonly command: string; + readonly workingDirectory?: string; + readonly env?: Record; + readonly abortSignal?: AbortSignal; + }) => + spawnProcess(options).then((processHandle) => + runBoundary( + Effect.gen(function* () { + const [stdout, stderr, result] = yield* Effect.all( + [ + collectUtf8(processHandle.stdout, MAX_PROCESS_OUTPUT_BYTES), + collectUtf8(processHandle.stderr, MAX_PROCESS_OUTPUT_BYTES), + Effect.tryPromise({ + try: () => processHandle.wait(), + catch: (error) => + error instanceof OmpDockerSandboxError + ? error + : sandboxError("OMP sandbox command wait failed."), + }), + ], + { concurrency: "unbounded" }, + ); + return { + exitCode: result.exitCode, + stdout, + stderr, + }; + }).pipe( + Effect.tapError(() => + Effect.tryPromise({ + try: () => processHandle.kill(), + catch: () => sandboxError("OMP sandbox command kill failed."), + }).pipe(Effect.ignore), + ), + ), + options.abortSignal, + ), + ); + + const sandbox: SandboxSession = { + description: + "OMP eval Docker sandbox. Workdir /eval. Runtime /opt/omp-eval is read-only. Bridge port 4000 is published on 127.0.0.1 only.", + readFile: ({ path, abortSignal }) => + runBoundary( + readBinary(path).pipe( + Effect.map((bytes) => { + if (bytes === null) return null; + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + }), + ), + abortSignal, + ), + readBinaryFile: ({ path, abortSignal }) => runBoundary(readBinary(path), abortSignal), + readTextFile: ({ path, abortSignal, encoding, startLine, endLine }) => + runBoundary( + readBinary(path).pipe( + Effect.flatMap((bytes) => + bytes === null + ? Effect.succeed(null) + : decodeText(bytes, encoding).pipe( + Effect.map((text) => sliceTextLines(text, startLine, endLine)), + ), + ), + ), + abortSignal, + ), + writeFile: ({ path, content, abortSignal }) => + runBoundary( + readStreamBytes(content, MAX_FILE_BYTES).pipe( + Effect.flatMap((bytes) => writeBinary(path, bytes)), + ), + abortSignal, + ), + writeBinaryFile: ({ path, content, abortSignal }) => + runBoundary(writeBinary(path, content), abortSignal), + writeTextFile: ({ path, content, abortSignal, encoding }) => + runBoundary( + encodeText(content, encoding).pipe(Effect.flatMap((bytes) => writeBinary(path, bytes))), + abortSignal, + ), + spawn: spawnProcess, + run: runCommand, + }; + + return { + ...sandbox, + id: container.id, + defaultWorkingDirectory: OMP_SANDBOX_WORKDIR, + ports: [DEFAULT_OMP_SANDBOX_PORT], + getPortEndpoint: ({ port, protocol }) => + runBoundary(inspectPublishedPort(container, port, protocol)), + getPortUrl: ({ port, protocol }) => + runBoundary( + inspectPublishedPort(container, port, protocol).pipe( + Effect.map((endpoint) => endpoint.url), + ), + ), + stop: close, + destroy: close, + restricted: () => sandbox, + }; +}; diff --git a/packages/evals/src/omp-guard.ts b/packages/evals/src/omp-guard.ts new file mode 100644 index 0000000..79141c6 --- /dev/null +++ b/packages/evals/src/omp-guard.ts @@ -0,0 +1,866 @@ +import { SKILL_NAMES, type SkillName } from "@askgina/contracts"; + +import { isUnknownRecord } from "./type-guards"; + +export const OMP_HARNESS_PROFILE = "omp-18.1.14-acp-v1" as const; + +const MAX_TOOL_NAMES = 128; +const MAX_TOOL_NAME_LENGTH = 64; +const MAX_READ_PATHS = 64; +const MAX_PATH_LENGTH = 4_096; +const MAX_BLOCKED_ACTIONS = 256; +const MAX_NATIVE_CALL_ID_LENGTH = 256; +const TOOL_NAME = /^[A-Za-z0-9_.-]+$/u; +const SKILL_URI = /^skill:\/\/([a-z0-9-]+)$/u; +const ASK_GINA_SKILL_NAMES: Readonly> = { + "review-gina-account": true, + "research-spot-tokens": true, + "research-hyperliquid": true, + "research-prediction-markets": true, +}; +const TOP_LEVEL_KEYS: Readonly> = { + version: true, + profile: true, + phase: true, + expectedTools: true, + activeTools: true, + inventoryExact: true, + activatedSkills: true, + blockedActions: true, + nativeCalls: true, + terminal: true, +}; +const NATIVE_CALL_KEYS: Readonly> = { + id: true, + name: true, +}; +const TERMINAL_KEYS: Readonly> = { + stopReason: true, + isError: true, + usage: true, +}; +const USAGE_KEYS: Readonly> = { + inputTokens: true, + outputTokens: true, + totalTokens: true, +}; + +export type OmpGuardPhase = "loaded" | "ready" | "terminal"; +export type OmpGuardStopReason = "stop" | "length" | "toolUse" | "aborted" | "error"; + +export interface OmpGuardUsageEvidence { + readonly inputTokens: number; + readonly outputTokens: number; + readonly totalTokens: number; +} + +export interface OmpGuardTerminalEvidence { + readonly stopReason: OmpGuardStopReason; + readonly isError: boolean; + readonly usage?: OmpGuardUsageEvidence; +} + +export interface OmpGuardNativeCallEvidence { + readonly id: string; + readonly name: string; +} + +export interface OmpGuardEvidence { + readonly version: 1; + readonly profile: typeof OMP_HARNESS_PROFILE; + readonly phase: OmpGuardPhase; + readonly expectedTools: readonly string[]; + readonly activeTools: readonly string[]; + readonly inventoryExact: boolean; + readonly activatedSkills: readonly SkillName[]; + readonly blockedActions: number; + readonly nativeCalls: readonly OmpGuardNativeCallEvidence[]; + readonly terminal?: OmpGuardTerminalEvidence; +} + +export interface OmpGuardExtensionSourceOptions { + readonly expectedTools: readonly string[]; + readonly allowedSkillUris: readonly string[]; + readonly evidencePath: string; + readonly allowedReadPaths?: readonly string[]; +} + +const hasOnlyKeys = ( + value: Readonly>, + allowed: Readonly>, +): boolean => Object.keys(value).every((key) => allowed[key] === true); + +const decodeBoundedStrings = ( + value: unknown, + maximumCount: number, + maximumLength: number, + allowEmpty: boolean, +): readonly string[] | undefined => { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > maximumCount) { + return undefined; + } + const names: string[] = []; + const seen = new Set(); + for (const entry of value) { + if ( + typeof entry !== "string" || + entry.length === 0 || + entry.length > maximumLength || + entry.trim() !== entry || + seen.has(entry) + ) { + return undefined; + } + seen.add(entry); + names.push(entry); + } + return names; +}; + +const decodeNativeCalls = (value: unknown): readonly OmpGuardNativeCallEvidence[] | undefined => { + if (!Array.isArray(value) || value.length > MAX_BLOCKED_ACTIONS) return undefined; + const calls: OmpGuardNativeCallEvidence[] = []; + const seenIds = new Set(); + for (const entry of value) { + if (!isUnknownRecord(entry) || !hasOnlyKeys(entry, NATIVE_CALL_KEYS)) return undefined; + const { id, name } = entry; + if ( + typeof id !== "string" || + id.length === 0 || + id.length > MAX_NATIVE_CALL_ID_LENGTH || + seenIds.has(id) || + typeof name !== "string" || + name.length === 0 || + name.length > MAX_TOOL_NAME_LENGTH + ) { + return undefined; + } + seenIds.add(id); + calls.push({ id, name }); + } + return calls; +}; + +const isExpectedToolInventory = (names: readonly string[]): boolean => + names.includes("read") && names.every((name) => TOOL_NAME.test(name)); + +const inventoriesMatch = (left: readonly string[], right: readonly string[]): boolean => { + if (left.length !== right.length) return false; + const rightNames = new Set(right); + return left.every((name) => rightNames.has(name)); +}; + +const isNonNegativeSafeInteger = (value: unknown): value is number => + typeof value === "number" && Number.isSafeInteger(value) && value >= 0; + +const decodeUsage = (value: unknown): OmpGuardUsageEvidence | undefined => { + if (!isUnknownRecord(value) || !hasOnlyKeys(value, USAGE_KEYS)) return undefined; + const { inputTokens, outputTokens, totalTokens } = value; + if ( + !isNonNegativeSafeInteger(inputTokens) || + !isNonNegativeSafeInteger(outputTokens) || + !isNonNegativeSafeInteger(totalTokens) + ) { + return undefined; + } + const conversationTokens = inputTokens + outputTokens; + if (!Number.isSafeInteger(conversationTokens) || totalTokens < conversationTokens) { + return undefined; + } + return { inputTokens, outputTokens, totalTokens }; +}; + +const decodeTerminal = (value: unknown): OmpGuardTerminalEvidence | undefined => { + if (!isUnknownRecord(value) || !hasOnlyKeys(value, TERMINAL_KEYS)) return undefined; + const { stopReason, isError } = value; + if ( + stopReason !== "stop" && + stopReason !== "length" && + stopReason !== "toolUse" && + stopReason !== "aborted" && + stopReason !== "error" + ) { + return undefined; + } + if (typeof isError !== "boolean") return undefined; + if (isError !== (stopReason === "aborted" || stopReason === "error")) return undefined; + if (Object.hasOwn(value, "usage")) { + const usage = decodeUsage(value.usage); + if (usage === undefined) return undefined; + return { stopReason, isError, usage }; + } + return { stopReason, isError }; +}; + +export const decodeOmpGuardEvidence = (value: unknown): OmpGuardEvidence | undefined => { + if (!isUnknownRecord(value) || !hasOnlyKeys(value, TOP_LEVEL_KEYS)) return undefined; + if (value.version !== 1 || value.profile !== OMP_HARNESS_PROFILE) return undefined; + if (value.phase !== "loaded" && value.phase !== "ready" && value.phase !== "terminal") { + return undefined; + } + const expectedTools = decodeBoundedStrings( + value.expectedTools, + MAX_TOOL_NAMES, + MAX_TOOL_NAME_LENGTH, + false, + ); + const activeTools = decodeBoundedStrings( + value.activeTools, + MAX_TOOL_NAMES, + MAX_TOOL_NAME_LENGTH, + true, + ); + if ( + expectedTools === undefined || + activeTools === undefined || + !isExpectedToolInventory(expectedTools) || + typeof value.inventoryExact !== "boolean" || + value.inventoryExact !== inventoriesMatch(expectedTools, activeTools) + ) { + return undefined; + } + const activated = decodeBoundedStrings(value.activatedSkills, SKILL_NAMES.length, 128, true); + const nativeCalls = decodeNativeCalls(value.nativeCalls); + if ( + activated === undefined || + activated.some((name) => ASK_GINA_SKILL_NAMES[name as SkillName] !== true) || + !isNonNegativeSafeInteger(value.blockedActions) || + value.blockedActions > MAX_BLOCKED_ACTIONS || + nativeCalls === undefined + ) { + return undefined; + } + const activatedSkills = activated as readonly SkillName[]; + const hasTerminal = Object.hasOwn(value, "terminal"); + + if (value.phase === "loaded") { + if ( + hasTerminal || + value.inventoryExact || + activeTools.length !== 0 || + activatedSkills.length !== 0 || + value.blockedActions !== 0 || + nativeCalls.length !== 0 + ) { + return undefined; + } + return { + version: 1, + profile: OMP_HARNESS_PROFILE, + phase: "loaded", + expectedTools, + activeTools, + inventoryExact: false, + activatedSkills, + blockedActions: 0, + nativeCalls, + }; + } + + if (value.phase === "ready") { + if (hasTerminal || !value.inventoryExact || nativeCalls.length !== 0) return undefined; + return { + version: 1, + profile: OMP_HARNESS_PROFILE, + phase: "ready", + expectedTools, + activeTools, + inventoryExact: true, + activatedSkills, + blockedActions: value.blockedActions, + nativeCalls, + }; + } + + if (!hasTerminal) return undefined; + const terminal = decodeTerminal(value.terminal); + if (terminal === undefined) return undefined; + if ( + !value.inventoryExact && + (terminal.stopReason !== "error" || !terminal.isError || terminal.usage !== undefined) + ) { + return undefined; + } + return { + version: 1, + profile: OMP_HARNESS_PROFILE, + phase: "terminal", + expectedTools, + activeTools, + inventoryExact: value.inventoryExact, + activatedSkills, + blockedActions: value.blockedActions, + nativeCalls, + terminal, + }; +}; + +const requireBoundedStrings = ( + value: readonly string[], + maximumCount: number, + maximumLength: number, + allowEmpty: boolean, +): readonly string[] => { + const decoded = decodeBoundedStrings(value, maximumCount, maximumLength, allowEmpty); + if (decoded === undefined) throw new Error("Invalid OMP guard configuration"); + return decoded; +}; + +export const makeOmpGuardedAcpLauncherSource = (expectedTools: readonly string[]): string => { + const tools = requireBoundedStrings(expectedTools, MAX_TOOL_NAMES, MAX_TOOL_NAME_LENGTH, false); + if (!isExpectedToolInventory(tools)) { + throw new Error("Invalid OMP guard configuration"); + } + + return `#!/usr/local/bin/node +import { spawn } from "node:child_process"; +import { readFileSync, unlinkSync } from "node:fs"; + +const NATIVE_EXECUTABLE = "/opt/omp-eval/omp"; +const EVIDENCE_PATH = "/eval/omp-eval-evidence.json"; +const PROFILE = ${JSON.stringify(OMP_HARNESS_PROFILE)}; +const EXPECTED_TOOLS = ${JSON.stringify(tools)}; +const MAX_LINE_BYTES = 8 * 1024 * 1024; +const MAX_EVIDENCE_BYTES = 65_536; +const FAILURE_EXIT_CODE = 86; +const SIGNAL_EXIT_CODES = { SIGHUP: 129, SIGINT: 130, SIGTERM: 143 }; +const FORWARDED_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"]; +let failed = false; +let settled = false; +let admitted = false; +let child; +let forceTimer; + +function failClosed() { + if (failed) return; + failed = true; + try { process.stderr.write("OMP evaluation launcher failed\\n"); } catch {} + try { process.stdin.pause(); } catch {} + if (child === undefined) { + process.exit(FAILURE_EXIT_CODE); + return; + } + try { child.stdin.end(); } catch {} + if (child.exitCode === null && child.signalCode === null) { + try { child.kill("SIGTERM"); } catch {} + forceTimer = setTimeout(() => { + try { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } catch {} + }, 750); + } +} + +function exitFromChild(code, signal) { + if (forceTimer !== undefined) clearTimeout(forceTimer); + for (const name of FORWARDED_SIGNALS) process.removeAllListeners(name); + if (failed) process.exit(FAILURE_EXIT_CODE); + if (signal !== null) process.exit(SIGNAL_EXIT_CODES[signal] ?? FAILURE_EXIT_CODE); + process.exit(code ?? FAILURE_EXIT_CODE); +} + +function guardLoaded() { + try { + const bytes = readFileSync(EVIDENCE_PATH); + if (bytes.length > MAX_EVIDENCE_BYTES) return false; + const value = JSON.parse(bytes.toString("utf8")); + return value !== null && typeof value === "object" && !Array.isArray(value) && + value.version === 1 && + value.profile === PROFILE && + value.phase === "loaded" && + value.inventoryExact === false && + value.blockedActions === 0 && + Array.isArray(value.activeTools) && value.activeTools.length === 0 && + Array.isArray(value.activatedSkills) && value.activatedSkills.length === 0 && + Array.isArray(value.nativeCalls) && value.nativeCalls.length === 0 && + !Object.hasOwn(value, "terminal") && + Array.isArray(value.expectedTools) && + value.expectedTools.length === EXPECTED_TOOLS.length && + EXPECTED_TOOLS.every((name, index) => value.expectedTools[index] === name); + } catch { + return false; + } +} + +function parseFrame(frame) { + let end = frame.length; + if (end > 0 && frame[end - 1] === 0x0a) end -= 1; + if (end > 0 && frame[end - 1] === 0x0d) end -= 1; + const value = JSON.parse(frame.toString("utf8", 0, end)); + if (value === null || typeof value !== "object" || Array.isArray(value) || value.jsonrpc !== "2.0") { + throw new Error("malformed"); + } + const response = Object.hasOwn(value, "id") && + (Object.hasOwn(value, "result") !== Object.hasOwn(value, "error")); + if (typeof value.method !== "string" && !response) throw new Error("malformed"); + return value; +} + +function writeFrame(frame) { + if (child.stdin.write(frame)) return Promise.resolve(); + const { promise, resolve, reject } = Promise.withResolvers(); + const onDrain = () => { + child.stdin.off("error", onError); + resolve(); + }; + const onError = (error) => { + child.stdin.off("drain", onDrain); + reject(error); + }; + child.stdin.once("drain", onDrain); + child.stdin.once("error", onError); + return promise; +} + +async function admitFrame(frame) { + const value = parseFrame(frame); + if (value.method !== "session/prompt") return; + if (!guardLoaded()) throw new Error("guard"); + admitted = true; +} + +async function forwardInput() { + let fragments = []; + let fragmentBytes = 0; + for await (const chunk of process.stdin) { + if (admitted) { + await writeFrame(chunk); + continue; + } + let start = 0; + while (start < chunk.length) { + const newline = chunk.indexOf(0x0a, start); + if (newline === -1) break; + const segment = chunk.subarray(start, newline + 1); + const frameBytes = fragmentBytes + segment.length; + if (frameBytes > MAX_LINE_BYTES) throw new Error("rejected"); + const frame = fragments.length === 0 ? segment : Buffer.concat([...fragments, segment], frameBytes); + fragments = []; + fragmentBytes = 0; + await admitFrame(frame); + await writeFrame(frame); + start = newline + 1; + if (admitted) { + if (start < chunk.length) await writeFrame(chunk.subarray(start)); + break; + } + } + if (!admitted && start < chunk.length) { + const fragment = chunk.subarray(start); + fragmentBytes += fragment.length; + if (fragmentBytes > MAX_LINE_BYTES) throw new Error("rejected"); + fragments.push(fragment); + } + } + if (fragmentBytes > 0) throw new Error("rejected"); + child.stdin.end(); +} + +try { + unlinkSync(EVIDENCE_PATH); +} catch (error) { + if (error === null || typeof error !== "object" || error.code !== "ENOENT") failClosed(); +} + +if (!failed) { + child = spawn(NATIVE_EXECUTABLE, process.argv.slice(2), { + detached: false, + env: process.env, + stdio: ["pipe", "inherit", "inherit"], + }); + for (const signal of FORWARDED_SIGNALS) { + process.on(signal, () => { + if (child.exitCode === null && child.signalCode === null) { + try { child.kill(signal); } catch {} + if (forceTimer === undefined) { + forceTimer = setTimeout(() => { + try { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + } catch {} + }, 750); + } + } + }); + } + child.stdin.on("error", failClosed); + child.on("error", failClosed); + child.on("close", (code, signal) => { + if (settled) return; + settled = true; + exitFromChild(code, signal); + }); + process.stdin.on("error", failClosed); + forwardInput().catch(failClosed); +} +`; +}; + +export const makeOmpGuardExtensionSource = (options: OmpGuardExtensionSourceOptions): string => { + const expectedTools = requireBoundedStrings( + options.expectedTools, + MAX_TOOL_NAMES, + MAX_TOOL_NAME_LENGTH, + false, + ); + const allowedSkillUris = requireBoundedStrings( + options.allowedSkillUris, + SKILL_NAMES.length, + MAX_PATH_LENGTH, + false, + ); + const allowedReadPaths = requireBoundedStrings( + options.allowedReadPaths ?? [], + MAX_READ_PATHS, + MAX_PATH_LENGTH, + true, + ); + if ( + !isExpectedToolInventory(expectedTools) || + typeof options.evidencePath !== "string" || + options.evidencePath.length === 0 || + options.evidencePath.length > MAX_PATH_LENGTH || + !options.evidencePath.startsWith("/") || + options.evidencePath.includes("\0") || + allowedSkillUris.length !== SKILL_NAMES.length || + allowedReadPaths.some( + (path) => !path.startsWith("/") || path.includes("\0") || path === options.evidencePath, + ) + ) { + throw new Error("Invalid OMP guard configuration"); + } + + const uriSkills: Record = {}; + const remainingSkills: Record = { ...ASK_GINA_SKILL_NAMES }; + for (const uri of allowedSkillUris) { + const match = SKILL_URI.exec(uri); + const skill = match?.[1]; + if (skill === undefined || remainingSkills[skill] !== true) { + throw new Error("Invalid OMP guard configuration"); + } + uriSkills[uri] = skill as SkillName; + delete remainingSkills[skill]; + } + if (Object.keys(remainingSkills).length !== 0) { + throw new Error("Invalid OMP guard configuration"); + } + + const pathSkills: Record = {}; + for (const readPath of allowedReadPaths) { + for (const skill of SKILL_NAMES) { + if (readPath.endsWith(`/${skill}/SKILL.md`)) { + pathSkills[readPath] = skill; + break; + } + } + } + + const configuration = JSON.stringify({ + expectedTools, + allowedSkillUris, + allowedReadPaths, + uriSkills, + pathSkills, + evidencePath: options.evidencePath, + }); + + return `import { renameSync, unlinkSync, writeFileSync } from "node:fs"; + +const PROFILE = ${JSON.stringify(OMP_HARNESS_PROFILE)}; +const CONFIG = ${configuration}; +const MAX_ATTEMPTS = ${MAX_BLOCKED_ACTIONS}; +const MAX_ID_LENGTH = ${MAX_NATIVE_CALL_ID_LENGTH}; +const MAX_EVIDENCE_BYTES = 65_536; +const FAILURE_EXIT_CODE = 86; +const expectedSet = new Set(CONFIG.expectedTools); +const skillUris = new Map(Object.entries(CONFIG.uriSkills)); +const readPaths = new Set(CONFIG.allowedReadPaths); +const pathSkills = new Map(Object.entries(CONFIG.pathSkills)); +const attempts = new Map(); +const state = { + version: 1, + profile: PROFILE, + phase: "loaded", + expectedTools: [...CONFIG.expectedTools], + activeTools: [], + inventoryExact: false, + activatedSkills: [], + blockedActions: 0, + nativeCalls: [], +}; +let admitted = false; +let stopping = false; +let assistantTurns = 0; +let usageAvailable = true; +let inputTokens = 0; +let outputTokens = 0; +let totalTokens = 0; + +function rawPersist() { + const temporaryPath = CONFIG.evidencePath + ".tmp"; + try { + const encoded = JSON.stringify(state); + if (Buffer.byteLength(encoded, "utf8") > MAX_EVIDENCE_BYTES) return false; + writeFileSync(temporaryPath, encoded + "\\n", { encoding: "utf8", flag: "w", mode: 0o600 }); + renameSync(temporaryPath, CONFIG.evidencePath); + return true; + } catch { + try { unlinkSync(temporaryPath); } catch {} + return false; + } +} + +function stopNow() { + if (!stopping) { + stopping = true; + admitted = false; + state.phase = "terminal"; + state.terminal = { stopReason: "error", isError: true }; + state.nativeCalls = Array.from(attempts, ([id, attempt]) => ({ id, name: attempt.name })); + rawPersist(); + } + process.exit(FAILURE_EXIT_CODE); + throw new Error("OMP evaluation guard stopped"); +} + +function persist() { + if (!rawPersist()) stopNow(); +} + +function validNames(value) { + if (!Array.isArray(value) || value.length === 0 || value.length > ${MAX_TOOL_NAMES}) return false; + const seen = new Set(); + for (const name of value) { + if ( + typeof name !== "string" || + name.length === 0 || + name.length > ${MAX_TOOL_NAME_LENGTH} || + name.trim() !== name || + seen.has(name) + ) return false; + seen.add(name); + } + return true; +} + +function sameInventory(left, right) { + if (left.length !== right.length) return false; + const names = new Set(right); + return left.every((name) => names.has(name)); +} + +function checkInventory(pi) { + admitted = false; + const active = pi.getActiveTools(); + if (!validNames(active) || !active.includes("read")) stopNow(); + state.activeTools = [...active]; + state.inventoryExact = sameInventory(state.expectedTools, state.activeTools); + if (!state.inventoryExact) stopNow(); + state.phase = "ready"; + delete state.terminal; + persist(); + admitted = true; +} + +function readPolicy(input) { + if (!input || typeof input !== "object" || Array.isArray(input)) { + return { allowed: false, activation: null, key: "blocked" }; + } + const target = input.path; + if (typeof target !== "string") return { allowed: false, activation: null, key: "blocked" }; + const uriSkill = skillUris.get(target); + if (uriSkill !== undefined) return { allowed: true, activation: uriSkill, key: "skill:" + target }; + if (!readPaths.has(target)) return { allowed: false, activation: null, key: "blocked" }; + const pathSkill = pathSkills.get(target) ?? null; + return { allowed: true, activation: pathSkill, key: "path:" + target }; +} + +function policyFor(name, input) { + if (!admitted || !state.inventoryExact || !expectedSet.has(name)) { + return { allowed: false, activation: null, key: "blocked" }; + } + return name === "read" + ? readPolicy(input) + : { allowed: true, activation: null, key: "tool:" + name }; +} + +function observeAttempt(id, name, input, source) { + if ( + typeof id !== "string" || + id.length === 0 || + id.length > MAX_ID_LENGTH || + typeof name !== "string" || + name.length === 0 || + name.length > ${MAX_TOOL_NAME_LENGTH} + ) stopNow(); + const policy = policyFor(name, input); + const prior = attempts.get(id); + if (prior !== undefined) { + if (prior.name !== name || prior.key !== policy.key || prior[source]) stopNow(); + prior[source] = true; + return prior; + } + if (attempts.size >= MAX_ATTEMPTS) stopNow(); + const observed = { + name, + allowed: policy.allowed, + activation: policy.activation, + key: policy.key, + hook: source === "hook", + message: source === "message", + result: false, + }; + attempts.set(id, observed); + if (!observed.allowed) state.blockedActions += 1; + return observed; +} + +function consumeUsage(value) { + assistantTurns += 1; + if (assistantTurns > MAX_ATTEMPTS) stopNow(); + if (!value || typeof value !== "object" || Array.isArray(value)) { + usageAvailable = false; + return; + } + const input = value.input; + const output = value.output; + const total = value.totalTokens; + if ( + !Number.isSafeInteger(input) || input < 0 || + !Number.isSafeInteger(output) || output < 0 || + !Number.isSafeInteger(total) || total < 0 || + !Number.isSafeInteger(input + output) || total < input + output + ) { + usageAvailable = false; + return; + } + if (!usageAvailable) return; + const nextInput = inputTokens + input; + const nextOutput = outputTokens + output; + const nextTotal = totalTokens + total; + if ( + !Number.isSafeInteger(nextInput) || + !Number.isSafeInteger(nextOutput) || + !Number.isSafeInteger(nextTotal) + ) { + usageAvailable = false; + return; + } + inputTokens = nextInput; + outputTokens = nextOutput; + totalTokens = nextTotal; +} + +function scanAssistantMessage(message) { + if (!message || typeof message !== "object" || Array.isArray(message)) stopNow(); + consumeUsage(message.usage); + if (!Array.isArray(message.content)) stopNow(); + for (const block of message.content) { + if (!block || typeof block !== "object" || Array.isArray(block) || block.type !== "toolCall") continue; + observeAttempt(block.id, block.name, block.arguments, "message"); + } + persist(); +} + +export default function ompEvalGuard(pi) { + try { + pi.on("before_agent_start", async () => { + try { + // ACP starts MCP discovery with the first prompt. + const readyBy = performance.now() + 5_000; + while (true) { + await pi.setActiveTools(CONFIG.expectedTools); + const active = pi.getActiveTools(); + if ( + !validNames(active) || !active.includes("read") || + active.some((name) => !expectedSet.has(name)) + ) stopNow(); + if (active.length === CONFIG.expectedTools.length || performance.now() >= readyBy) break; + const { promise, resolve } = Promise.withResolvers(); + setTimeout(resolve, 25); + await promise; + } + checkInventory(pi); + } catch { stopNow(); } + }); + pi.on("before_provider_request", (event) => { + try { + if (!admitted) stopNow(); + checkInventory(pi); + return event.payload; + } catch { stopNow(); } + }); + pi.on("tool_call", (event) => { + try { + const attempt = observeAttempt(event.toolCallId, event.toolName, event.input, "hook"); + persist(); + if (!attempt.allowed) return { block: true, reason: "Tool blocked by evaluation guard" }; + } catch { stopNow(); } + }); + pi.on("message_end", (event) => { + try { + if (event.message && event.message.role === "assistant") scanAssistantMessage(event.message); + } catch { stopNow(); } + }); + pi.on("tool_result", (event) => { + try { + const attempt = attempts.has(event.toolCallId) + ? attempts.get(event.toolCallId) + : observeAttempt(event.toolCallId, event.toolName, event.input, "hook"); + if ( + attempt.name !== event.toolName || + attempt.result || + typeof event.isError !== "boolean" + ) stopNow(); + attempt.result = true; + if ( + !event.isError && + attempt.allowed && + attempt.activation !== null && + !state.activatedSkills.includes(attempt.activation) + ) { + state.activatedSkills.push(attempt.activation); + } + persist(); + } catch { stopNow(); } + }); + pi.on("agent_end", (event) => { + try { + if (event.willContinue === true) { + persist(); + return; + } + if (!Array.isArray(event.messages)) stopNow(); + let lastAssistant; + for (let index = event.messages.length - 1; index >= 0; index -= 1) { + const message = event.messages[index]; + if (message && typeof message === "object" && message.role === "assistant") { + lastAssistant = message; + break; + } + } + const reason = lastAssistant && lastAssistant.stopReason; + if ( + reason !== "stop" && reason !== "length" && reason !== "toolUse" && + reason !== "aborted" && reason !== "error" + ) stopNow(); + const terminal = { + stopReason: reason, + isError: reason === "aborted" || reason === "error", + }; + if (assistantTurns > 0 && usageAvailable) { + terminal.usage = { inputTokens, outputTokens, totalTokens }; + } + admitted = false; + state.phase = "terminal"; + state.terminal = terminal; + state.nativeCalls = Array.from(attempts, ([id, attempt]) => ({ id, name: attempt.name })); + persist(); + } catch { stopNow(); } + }); + persist(); + } catch { + stopNow(); + } +} +`; +}; diff --git a/packages/evals/src/omp-harness.ts b/packages/evals/src/omp-harness.ts new file mode 100644 index 0000000..371bc7b --- /dev/null +++ b/packages/evals/src/omp-harness.ts @@ -0,0 +1,1810 @@ +import { createHash } from "node:crypto"; +import { createMCPClient, type ListToolsResult, type MCPClient } from "@ai-sdk/mcp"; +import Ajv, { type ValidateFunction } from "ajv"; +import addFormats from "ajv-formats"; +import { HarnessAgent, type HarnessAgentSession } from "@ai-sdk/harness/agent"; +import { createACP } from "@ai-sdk/harness-acp"; +import { createCredentialRequestTransformation } from "@ai-sdk/harness/utils"; +import type { HarnessV1SandboxProvider } from "@ai-sdk/harness"; +import { + listCatalogToolNames, + PRODUCTION_MCP_URL, + SKILL_NAMES, + type SkillName, +} from "@askgina/contracts"; +import { jsonSchema, type StepResult, type ToolSet } from "ai"; +import { + Clock, + Data, + DateTime, + Duration, + Effect, + Exit, + FileSystem, + Function, + Option, + Path, + Redacted, + Scope, +} from "effect"; + +import type { PluginEvalCase, PluginEvalObservation, PluginEvalToolCall } from "./contracts"; +import { createOmpDockerSandbox } from "./omp-docker-sandbox"; +import { + decodeOmpGuardEvidence, + makeOmpGuardExtensionSource, + makeOmpGuardedAcpLauncherSource, + type OmpGuardEvidence, +} from "./omp-guard"; + +const DEFAULT_TIMEOUT_MS = 120_000; +const MAX_MCP_TOOL_PAGES = 32; +const MAX_MCP_CLOSE_WAIT_MS = 1_000; +const MAX_SESSION_DESTROY_WAIT_MS = 8_000; +const OMP_REQUIRED_VERSION = "18.1.14"; +const OMP_INCOMPLETE_GENERATION_ERROR = "OMP generation did not complete with a final answer"; +const OMP_POLICY_ERROR = "OMP used an action outside approved skill reads or canonical Gina reads"; +const OMP_TOOL_EXECUTION_ERROR = "OMP tool execution failed"; +const HOST_TOOL_MCP_SERVER_NAME = "ai-sdk-harness-tools"; +const CONTAINER_OMP_PATH = "/opt/omp-eval/omp"; +const CONTAINER_GUARD_PATH = "/opt/omp-eval/omp-eval-guard.mjs"; +const CONTAINER_ACP_LAUNCHER_PATH = "/opt/omp-eval/omp-eval-acp.mjs"; +const CONTAINER_CONFIG_PATH = "/opt/omp-eval/config.yml"; +const CONTAINER_EVIDENCE_PATH = "/eval/omp-eval-evidence.json"; +const OMP_EVAL_PROVIDER_ALIAS = "omp-eval"; +const OMP_EVAL_PROVIDER_API_KEY_ENV = "OMP_EVAL_PROVIDER_API_KEY"; +const CANONICAL_ALLOWED_TOOLS = listCatalogToolNames(); +const UTF8_ENCODER = new TextEncoder(); +const SHA256_HEX = /^[a-f0-9]{64}$/u; +const OMP_MODEL_THINKING_EFFORTS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const; +const PROVIDER_PROFILE = { + openai: { + baseUrl: "https://api.openai.com/v1", + api: "openai-completions", + thinkingMode: "effort", + thinkingFormat: "openai", + }, + anthropic: { + baseUrl: "https://api.anthropic.com", + api: "anthropic-messages", + thinkingMode: "budget", + }, + openrouter: { + baseUrl: "https://openrouter.ai/api/v1", + api: "openai-completions", + thinkingMode: "effort", + thinkingFormat: "openrouter", + }, +} as const; +const OMP_REASONING = { + off: true, + minimal: true, + low: true, + medium: true, + high: true, + xhigh: true, + max: true, + auto: true, +} as const; +const NATIVE_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/u; +const SKILL_FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/; +const MAX_MCP_TOOL_NAME_LENGTH = 64; +const MCP_TOOL_NAME_HASH_LENGTH = 8; +const CANONICAL_TOOL_NAMES: Record = Object.fromEntries( + CANONICAL_ALLOWED_TOOLS.map((name) => [name, true as const]), +); + +export type OmpProvider = keyof typeof PROVIDER_PROFILE; +export type OmpReasoning = keyof typeof OMP_REASONING; + +export const isOmpProvider = (value: string): value is OmpProvider => + Object.hasOwn(PROVIDER_PROFILE, value); + +const isOmpReasoning = (value: string): value is OmpReasoning => + Object.hasOwn(OMP_REASONING, value); + +export interface PrepareOmpHarnessRuntimeOptions { + readonly root: string; + readonly executablePath: string; + readonly expectedSha256: string; +} + +export interface PreparedOmpHarnessRuntime { + readonly runtimeDirectory: string; +} + +export interface OmpHarnessTrialOptions { + readonly runId: string; + readonly repetition: number; + readonly availableTools: readonly string[]; + readonly runtimeDirectory: string; + readonly provider: OmpProvider; + readonly model: string; + readonly reasoning: string; + readonly apiKey: Redacted.Redacted; + readonly mcpAuthorization: Redacted.Redacted; + readonly timeoutMs: number; + readonly serverUrl?: string; + readonly providerBaseUrl?: string; + readonly dockerImage?: string; + readonly dockerNetwork?: string; + readonly sandbox?: HarnessV1SandboxProvider; +} + +interface ValidatedOmpHarnessTrialOptions { + readonly runId: string; + readonly repetition: number; + readonly availableTools: readonly string[]; + readonly runtimeDirectory: string; + readonly provider: OmpProvider; + readonly model: string; + readonly modelIdentity: string; + readonly reasoning: OmpReasoning; + readonly apiKey: string; + readonly mcpAuthorization: string; + readonly timeoutMs: number; + readonly serverUrl: string; + readonly providerBaseUrl?: string; + readonly dockerImage?: string; + readonly dockerNetwork?: string; + readonly sandbox?: HarnessV1SandboxProvider; +} + +interface CapturedHostToolCall { + readonly toolCallId: string; + readonly name: string; + readonly arguments: PluginEvalToolCall["arguments"]; + readonly durationMs?: number; + readonly resultBytes?: number; + readonly error?: PluginEvalToolCall["error"]; +} + +interface NativeSdkToolCall { + readonly input: unknown; + outcome?: "result" | "error"; +} + +interface ObservedOmpToolCalls { + readonly toolCalls: readonly PluginEvalToolCall[]; + readonly failedNativeRead: boolean; +} + +interface EvidenceSandbox { + readonly readTextFile: (options: { readonly path: string }) => PromiseLike; + readonly run: (options: { + readonly command: string; + readonly abortSignal?: AbortSignal; + }) => PromiseLike<{ readonly exitCode: number; readonly stdout: string }>; +} + +export class PluginEvalOmpHarnessExecutableError extends Data.TaggedError( + "PluginEvalOmpHarnessExecutableError", +)<{ + readonly reason: "invalid-path" | "invalid-file" | "digest-mismatch" | "forbidden-path"; +}> {} + +export class PluginEvalOmpHarnessRequestError extends Data.TaggedError( + "PluginEvalOmpHarnessRequestError", +)<{ + readonly caseId: string; + readonly reason: "invalid-options" | "unsupported-reasoning" | "invalid-endpoint"; +}> {} + +export class PluginEvalOmpHarnessSpawnError extends Data.TaggedError( + "PluginEvalOmpHarnessSpawnError", +)<{ + readonly caseId: string; + readonly reason: "could_not_start" | "preflight-failed" | "unsupported-version"; +}> {} + +export class PluginEvalOmpHarnessMcpError extends Data.TaggedError("PluginEvalOmpHarnessMcpError")<{ + readonly caseId: string; + readonly reason: "connection-failed" | "catalog-failed" | "catalog-mismatch" | "cleanup-failed"; +}> {} + +export class PluginEvalOmpHarnessProcessError extends Data.TaggedError( + "PluginEvalOmpHarnessProcessError", +)<{ + readonly caseId: string; + readonly reason: "generation-failed" | "incomplete-evidence" | "inventory-mismatch"; +}> {} + +export class PluginEvalOmpHarnessTimeoutError extends Data.TaggedError( + "PluginEvalOmpHarnessTimeoutError", +)<{ + readonly caseId: string; + readonly timeoutMs: number; +}> {} + +export type PluginEvalOmpHarnessError = + | PluginEvalOmpHarnessExecutableError + | PluginEvalOmpHarnessRequestError + | PluginEvalOmpHarnessSpawnError + | PluginEvalOmpHarnessMcpError + | PluginEvalOmpHarnessProcessError + | PluginEvalOmpHarnessTimeoutError; + +const catalogsMatch = (left: readonly string[], right: readonly string[]): boolean => { + if (left.length !== right.length) return false; + const uniqueLeft = new Set(left); + return uniqueLeft.size === left.length && right.every((tool) => uniqueLeft.has(tool)); +}; + +const isWithin = (path: Path.Path, parent: string, child: string): boolean => { + const relative = path.relative(parent, child); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +}; + +const isNativeExecutableHeader = (bytes: readonly number[]): boolean => + (bytes[0] === 0x7f && bytes[1] === 0x45 && bytes[2] === 0x4c && bytes[3] === 0x46) || + (bytes[0] === 0x4d && bytes[1] === 0x5a) || + (bytes[0] === 0xcf && bytes[1] === 0xfa && bytes[2] === 0xed && bytes[3] === 0xfe) || + (bytes[0] === 0xfe && bytes[1] === 0xed && bytes[2] === 0xfa && bytes[3] === 0xcf); + +const sanitizeMcpToolNamePart = (value: string, fallback: string): string => { + const sanitized = value + .toLowerCase() + .replace(/[^a-z_]+/g, "_") + .replace(/_+/g, "_") + .replace(/^_+|_+$/g, ""); + return sanitized.length > 0 ? sanitized : fallback; +}; + +const capMcpToolNameLength = (name: string): string => { + if (name.length <= MAX_MCP_TOOL_NAME_LENGTH) return name; + const hash = Bun.hash(name).toString(36).slice(0, MCP_TOOL_NAME_HASH_LENGTH); + const keep = MAX_MCP_TOOL_NAME_LENGTH - hash.length - 1; + return `${name.slice(0, keep)}_${hash}`; +}; + +const createOmpNativeToolName = (canonicalName: string): string => { + const sanitizedServerName = sanitizeMcpToolNamePart(HOST_TOOL_MCP_SERVER_NAME, "server"); + const sanitizedToolName = sanitizeMcpToolNamePart(canonicalName, "tool"); + const prefixWithUnderscore = `${sanitizedServerName}_`; + const normalizedToolName = sanitizedToolName.startsWith(prefixWithUnderscore) + ? sanitizedToolName.slice(prefixWithUnderscore.length) + : sanitizedToolName; + return capMcpToolNameLength(`mcp__${sanitizedServerName}_${normalizedToolName}`); +}; + +const CANONICAL_TOOL_BY_NATIVE_NAME: Readonly> = Object.fromEntries( + CANONICAL_ALLOWED_TOOLS.map((canonicalName) => [ + createOmpNativeToolName(canonicalName), + canonicalName, + ]), +); + +export const OMP_HARNESS_EXPECTED_NATIVE_TOOLS: readonly string[] = [ + "read", + ...CANONICAL_ALLOWED_TOOLS.map(createOmpNativeToolName), +]; +const OMP_NATIVE_BUILTIN_INPUT_SCHEMA = jsonSchema>({ + type: "object", +}); +const OMP_NATIVE_TOOL_METADATA: ToolSet = Object.fromEntries( + OMP_HARNESS_EXPECTED_NATIVE_TOOLS.map((name) => [ + name, + { + nativeName: name, + title: name === "read" ? "skill://" : name, + inputSchema: OMP_NATIVE_BUILTIN_INPUT_SCHEMA, + }, + ]), +); +const ALLOWED_SKILL_URIS: readonly string[] = SKILL_NAMES.map((name) => `skill://${name}`); + +const isJsonValue = (value: unknown): boolean => { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) + ) { + return true; + } + if (Array.isArray(value)) return value.every(isJsonValue); + if (typeof value !== "object") return false; + return Object.values(value).every(isJsonValue); +}; + +const jsonObject = (value: unknown): PluginEvalToolCall["arguments"] | undefined => { + if (typeof value !== "object" || value === null || Array.isArray(value) || !isJsonValue(value)) { + return undefined; + } + return value as PluginEvalToolCall["arguments"]; +}; + +const resultByteLength = (value: unknown): number | undefined => { + try { + const serialized = typeof value === "string" ? value : JSON.stringify(value); + return serialized === undefined ? undefined : UTF8_ENCODER.encode(serialized).byteLength; + } catch { + return undefined; + } +}; + +const inventoryAdmitted = (evidence: OmpGuardEvidence | undefined): boolean => + evidence !== undefined && + evidence.inventoryExact === true && + evidence.phase !== "loaded" && + catalogsMatch(evidence.expectedTools, OMP_HARNESS_EXPECTED_NATIVE_TOOLS) && + catalogsMatch(evidence.activeTools, OMP_HARNESS_EXPECTED_NATIVE_TOOLS); + +const timeoutError = (caseId: string, timeoutMs: number): PluginEvalOmpHarnessTimeoutError => + new PluginEvalOmpHarnessTimeoutError({ caseId, timeoutMs }); + +const ensureBeforeDeadline = ( + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => + Effect.flatMap(Clock.currentTimeMillis, (now) => + now >= deadlineMillis ? Effect.fail(timeoutError(caseId, timeoutMs)) : Effect.void, + ); + +const withRunDeadline = ( + effect: Effect.Effect, + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => + Effect.gen(function* () { + const before = yield* Clock.currentTimeMillis; + if (before >= deadlineMillis) return yield* timeoutError(caseId, timeoutMs); + return yield* effect.pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(deadlineMillis - before), + orElse: () => Effect.fail(timeoutError(caseId, timeoutMs)), + }), + Effect.matchEffect({ + onFailure: (error) => + ensureBeforeDeadline(caseId, timeoutMs, deadlineMillis).pipe( + Effect.flatMap(() => Effect.fail(error)), + ), + onSuccess: (value) => + ensureBeforeDeadline(caseId, timeoutMs, deadlineMillis).pipe(Effect.as(value)), + }), + ); + }); + +const abortablePromise = ( + run: (signal: AbortSignal) => PromiseLike, + onError: () => E, +): Effect.Effect => + Effect.callback((resume, signal) => { + void Promise.resolve() + .then(() => run(signal)) + .then( + (value) => { + if (!signal.aborted) resume(Effect.succeed(value)); + }, + () => { + if (!signal.aborted) resume(Effect.fail(onError())); + }, + ); + }); + +const catalogFailed = (caseId: string): PluginEvalOmpHarnessMcpError => + new PluginEvalOmpHarnessMcpError({ caseId, reason: "catalog-failed" }); + +type McpCloseOutcome = "closed" | "failed"; +const mcpClientCloses = new WeakMap>(); + +const closeMcpClientOnce = (client: MCPClient): Promise => { + const activeClose = mcpClientCloses.get(client); + if (activeClose !== undefined) return activeClose; + let closeResult: PromiseLike; + try { + closeResult = client.close(); + } catch { + const failed = Promise.resolve("failed"); + mcpClientCloses.set(client, failed); + return failed; + } + const outcome = Promise.resolve(closeResult).then( + (): McpCloseOutcome => "closed", + (): McpCloseOutcome => "failed", + ); + mcpClientCloses.set(client, outcome); + return outcome; +}; + +type HarnessSessionDestroyOutcome = "destroyed" | "failed"; +type RawSandboxDestroy = () => PromiseLike; +type RawSandboxDestroyRef = { current: RawSandboxDestroy | undefined }; +const harnessSessionDestroys = new WeakMap< + HarnessAgentSession, + Promise +>(); + +const settleHarnessDestroy = ( + destroy: RawSandboxDestroy, +): Promise => { + try { + return Promise.resolve(destroy()).then( + (): HarnessSessionDestroyOutcome => "destroyed", + (): HarnessSessionDestroyOutcome => "failed", + ); + } catch { + return Promise.resolve("failed"); + } +}; + +const destroyHarnessSessionOnce = ( + session: HarnessAgentSession, + rawSandboxDestroy: RawSandboxDestroyRef, +): Promise => { + const activeDestroy = harnessSessionDestroys.get(session); + if (activeDestroy !== undefined) return activeDestroy; + const sessionDestroy = settleHarnessDestroy(() => session.destroy()); + const rawDestroy = + rawSandboxDestroy.current === undefined + ? Promise.resolve("failed") + : settleHarnessDestroy(rawSandboxDestroy.current); + const outcome = Promise.all([sessionDestroy, rawDestroy]).then( + (results): HarnessSessionDestroyOutcome => + results.every((result) => result === "destroyed") ? "destroyed" : "failed", + ); + harnessSessionDestroys.set(session, outcome); + return outcome; +}; + +const isKnownHarnessError = (error: unknown): error is PluginEvalOmpHarnessError => + error instanceof PluginEvalOmpHarnessExecutableError || + error instanceof PluginEvalOmpHarnessRequestError || + error instanceof PluginEvalOmpHarnessSpawnError || + error instanceof PluginEvalOmpHarnessMcpError || + error instanceof PluginEvalOmpHarnessProcessError || + error instanceof PluginEvalOmpHarnessTimeoutError; + +const parseProviderBaseUrl = (value: string): string | undefined => { + if (value.includes("@") || value.includes("\n") || value.includes("\0") || value.includes("\\")) { + return undefined; + } + try { + const url = new URL(value); + if ( + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username !== "" || + url.password !== "" || + url.href.includes("@") + ) { + return undefined; + } + return url.href.replace(/\/$/u, "") === value.replace(/\/$/u, "") ? value : url.href; + } catch { + return undefined; + } +}; + +const yamlQuote = (value: string): string => + /[:#|>*&!%@`'"]/u.test(value) || value !== value.trim() ? JSON.stringify(value) : value; + +const nestedEvalConfig = [ + "startup:", + " checkUpdate: false", + " quiet: true", + "marketplace:", + " autoUpdate: off", + "memory:", + " backend: off", + "memories:", + " enabled: false", + "advisor:", + " enabled: false", + "task:", + " isolation:", + " enabled: false", + " eager: default", + " batch: false", + "dev:", + " autoqa: false", + "tools:", + " intentTracing: false", + "retry:", + " enabled: false", + " modelFallback: false", + "exa:", + " enabled: false", + "async:", + " enabled: false", + "bash:", + " autoBackground:", + " enabled: false", + "eval:", + " autoBackground:", + " enabled: false", + "skills:", + " enabled: true", + " enableSkillCommands: false", + " enableCodexUser: false", + " enableClaudeUser: false", + " enableClaudeProject: false", + " enablePiUser: true", + " enablePiProject: false", + " enableAgentsUser: false", + " enableAgentsProject: false", + " includeSkills:", + ...SKILL_NAMES.map((name) => ` - ${name}`), + "", +].join("\n"); + +const modelsYaml = ( + provider: OmpProvider, + model: string, + providerBaseUrl: string | undefined, +): string => { + const profile = PROVIDER_PROFILE[provider]; + const lines = [ + "providers:", + ` ${OMP_EVAL_PROVIDER_ALIAS}:`, + ` baseUrl: ${yamlQuote(providerBaseUrl ?? profile.baseUrl)}`, + ` apiKey: ${OMP_EVAL_PROVIDER_API_KEY_ENV}`, + " auth: apiKey", + ` api: ${profile.api}`, + " models:", + ` - id: ${JSON.stringify(model)}`, + " reasoning: true", + " thinking:", + ` mode: ${profile.thinkingMode}`, + " efforts:", + ]; + for (const effort of OMP_MODEL_THINKING_EFFORTS) { + lines.push(` - ${effort}`); + } + if ("thinkingFormat" in profile) { + lines.push( + " compat:", + " supportsReasoningEffort: true", + ` thinkingFormat: ${profile.thinkingFormat}`, + ); + } + lines.push(""); + return lines.join("\n"); +}; + +const installCommand = ( + provider: OmpProvider, + model: string, + providerBaseUrl: string | undefined, +): string => { + const models = modelsYaml(provider, model, providerBaseUrl); + return [ + 'mkdir -p "$HOME/.local/bin" "$HOME/.omp/agent/skills"', + `ln -sfn ${CONTAINER_ACP_LAUNCHER_PATH} "$HOME/.local/bin/omp"`, + `cp ${CONTAINER_CONFIG_PATH} "$HOME/.omp/agent/config.yml"`, + "cat > \"$HOME/.omp/agent/models.yml\" <<'OMP_EVAL_MODELS_YML'", + models.trimEnd(), + "OMP_EVAL_MODELS_YML", + `version="$(${CONTAINER_OMP_PATH} --version)"`, + `[ "$version" = "omp/${OMP_REQUIRED_VERSION}" ] || exit 1`, + ].join("\n"); +}; + +const snapshotExecutable = ( + executablePath: string, + expectedSha256: string, + snapshotPath: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if (!path.isAbsolute(executablePath) || !SHA256_HEX.test(expectedSha256)) { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "invalid-path" }); + } + const canonical = yield* fs + .realPath(executablePath) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-path" })), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const source = yield* fs + .open(canonical, { flag: "r" }) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + const snapshotWriter = yield* fs + .open(snapshotPath, { flag: "wx", mode: 0o555 }) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + const before = yield* source.stat.pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + if (before.type !== "File" || (before.mode & 0o111) === 0) { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }); + } + yield* source + .seek(0n, "start") + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + const hash = createHash("sha256"); + const header: number[] = []; + while (true) { + const maybeChunk = yield* source + .readAlloc(64 * 1024) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + if (Option.isNone(maybeChunk)) break; + const chunk = maybeChunk.value; + hash.update(chunk); + yield* snapshotWriter + .writeAll(chunk) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + for (let index = 0; index < chunk.length && header.length < 4; index += 1) { + const value = chunk[index]; + if (value !== undefined) header.push(value); + } + } + yield* snapshotWriter.sync.pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + const after = yield* source.stat.pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + const sha256 = hash.digest("hex"); + const beforeIno = Option.getOrUndefined(before.ino); + const afterIno = Option.getOrUndefined(after.ino); + if ( + !isNativeExecutableHeader(header) || + sha256 !== expectedSha256 || + after.type !== "File" || + before.dev !== after.dev || + beforeIno !== afterIno || + before.mode !== after.mode || + before.size !== after.size + ) { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "digest-mismatch" }); + } + }), + ); + yield* fs + .chmod(snapshotPath, 0o555) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + }); + +const parseSkillFrontmatter = ( + content: string, + expectedName: SkillName, +): Effect.Effect<{ readonly description: string }, PluginEvalOmpHarnessExecutableError> => { + const block = SKILL_FRONTMATTER.exec(content)?.[1]; + if (block === undefined) { + return Effect.fail(new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })); + } + const record: Record = {}; + for (const line of block.split(/\r?\n/u)) { + const separator = line.indexOf(":"); + if (separator <= 0) continue; + record[line.slice(0, separator).trim()] = line.slice(separator + 1).trim(); + } + const description = record.description; + if (record.name !== expectedName || description === undefined || description.length === 0) { + return Effect.fail(new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })); + } + return Effect.succeed({ description }); +}; + +const stageCanonicalSkills = ( + root: string, + runtimeDirectory: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const canonicalRoot = yield* fs + .realPath(root) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-path" })), + ); + const skillsRoot = path.join(canonicalRoot, "plugins", "ask-gina", "skills"); + const skillsDestination = path.join(runtimeDirectory, "skills"); + yield* fs + .makeDirectory(skillsDestination, { recursive: true, mode: 0o755 }) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* fs + .chmod(skillsDestination, 0o755) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* Effect.forEach( + SKILL_NAMES, + (name) => + Effect.gen(function* () { + const candidate = path.join(skillsRoot, name, "SKILL.md"); + if (!isWithin(path, skillsRoot, candidate) || path.basename(candidate) !== "SKILL.md") { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "forbidden-path" }); + } + const realPath = yield* fs + .realPath(candidate) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + if (!isWithin(path, skillsRoot, realPath) || path.basename(realPath) !== "SKILL.md") { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "forbidden-path" }); + } + const content = yield* fs + .readFileString(realPath) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + yield* parseSkillFrontmatter(content, name); + const destinationDirectory = path.join(skillsDestination, name); + const destinationFile = path.join(destinationDirectory, "SKILL.md"); + yield* fs + .makeDirectory(destinationDirectory, { recursive: true, mode: 0o755 }) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + yield* fs + .chmod(destinationDirectory, 0o755) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + yield* fs + .writeFileString(destinationFile, content, { + flag: "wx", + mode: 0o444, + }) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + yield* fs + .chmod(destinationFile, 0o444) + .pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + ), + ); + }), + { concurrency: 1 }, + ); + }); + +export const prepareOmpHarnessRuntime = ( + options: PrepareOmpHarnessRuntimeOptions, +): Effect.Effect< + PreparedOmpHarnessRuntime, + PluginEvalOmpHarnessExecutableError, + FileSystem.FileSystem | Path.Path | Scope.Scope +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if (!path.isAbsolute(options.root) || !path.isAbsolute(options.executablePath)) { + return yield* new PluginEvalOmpHarnessExecutableError({ reason: "invalid-path" }); + } + const expectedSha256 = options.expectedSha256.toLowerCase(); + const runtimeDirectory = yield* fs + .makeTempDirectoryScoped({ prefix: "ask-gina-omp-runtime-" }) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* fs + .chmod(runtimeDirectory, 0o755) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* snapshotExecutable( + options.executablePath, + expectedSha256, + path.join(runtimeDirectory, "omp"), + ); + yield* stageCanonicalSkills(options.root, runtimeDirectory); + const configPath = path.join(runtimeDirectory, "config.yml"); + yield* fs + .writeFileString(configPath, nestedEvalConfig, { + flag: "wx", + mode: 0o444, + }) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* fs + .chmod(configPath, 0o444) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + const guardSource = yield* Effect.try({ + try: () => + makeOmpGuardExtensionSource({ + expectedTools: OMP_HARNESS_EXPECTED_NATIVE_TOOLS, + allowedSkillUris: ALLOWED_SKILL_URIS, + evidencePath: CONTAINER_EVIDENCE_PATH, + }), + catch: () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + }); + const guardPath = path.join(runtimeDirectory, "omp-eval-guard.mjs"); + yield* fs + .writeFileString(guardPath, guardSource, { + flag: "wx", + mode: 0o444, + }) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* fs + .chmod(guardPath, 0o444) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + const launcherSource = yield* Effect.try({ + try: () => makeOmpGuardedAcpLauncherSource(OMP_HARNESS_EXPECTED_NATIVE_TOOLS), + catch: () => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" }), + }); + const launcherPath = path.join(runtimeDirectory, "omp-eval-acp.mjs"); + yield* fs + .writeFileString(launcherPath, launcherSource, { + flag: "wx", + mode: 0o555, + }) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + yield* fs + .chmod(launcherPath, 0o555) + .pipe( + Effect.mapError(() => new PluginEvalOmpHarnessExecutableError({ reason: "invalid-file" })), + ); + return { runtimeDirectory }; + }); + +const validateOptions = ( + evalCase: PluginEvalCase, + options: OmpHarnessTrialOptions, +): Effect.Effect => { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const apiKey = Redacted.value(options.apiKey); + const mcpAuthorization = Redacted.value(options.mcpAuthorization); + const model = options.model.trim(); + const runId = options.runId.trim(); + const providerBaseUrl = + options.providerBaseUrl === undefined + ? undefined + : parseProviderBaseUrl(options.providerBaseUrl); + const modelIdentity = `${options.provider}/${model}`; + const serverUrl = options.serverUrl ?? PRODUCTION_MCP_URL; + if (options.providerBaseUrl !== undefined && providerBaseUrl === undefined) { + return Effect.fail( + new PluginEvalOmpHarnessRequestError({ caseId: evalCase.id, reason: "invalid-endpoint" }), + ); + } + if ( + !isOmpProvider(options.provider) || + !catalogsMatch(options.availableTools, CANONICAL_ALLOWED_TOOLS) || + runId.length === 0 || + model.length === 0 || + apiKey.trim().length === 0 || + mcpAuthorization.trim().length === 0 || + !options.runtimeDirectory.startsWith("/") || + !Number.isSafeInteger(options.repetition) || + options.repetition <= 0 || + !Number.isSafeInteger(timeoutMs) || + timeoutMs <= 0 || + (options.serverUrl !== undefined && options.serverUrl.trim().length === 0) + ) { + return Effect.fail( + new PluginEvalOmpHarnessRequestError({ caseId: evalCase.id, reason: "invalid-options" }), + ); + } + if (!isOmpReasoning(options.reasoning)) { + return Effect.fail( + new PluginEvalOmpHarnessRequestError({ + caseId: evalCase.id, + reason: "unsupported-reasoning", + }), + ); + } + return Effect.succeed({ + runId, + repetition: options.repetition, + availableTools: [...CANONICAL_ALLOWED_TOOLS], + runtimeDirectory: options.runtimeDirectory, + provider: options.provider, + model, + modelIdentity, + reasoning: options.reasoning, + apiKey, + mcpAuthorization, + timeoutMs, + serverUrl, + ...(providerBaseUrl === undefined ? {} : { providerBaseUrl }), + ...(options.dockerImage === undefined ? {} : { dockerImage: options.dockerImage }), + ...(options.dockerNetwork === undefined ? {} : { dockerNetwork: options.dockerNetwork }), + ...(options.sandbox === undefined ? {} : { sandbox: options.sandbox }), + }); +}; + +const listAllMcpTools = ( + client: MCPClient, + caseId: string, +): Effect.Effect => + Effect.gen(function* () { + let page = yield* abortablePromise( + (signal) => client.listTools({ options: { signal } }), + () => catalogFailed(caseId), + ); + const tools = [...page.tools]; + const seenCursors = new Set(); + let pageCount = 1; + while (page.nextCursor !== undefined) { + if (pageCount >= MAX_MCP_TOOL_PAGES || seenCursors.has(page.nextCursor)) { + return yield* catalogFailed(caseId); + } + seenCursors.add(page.nextCursor); + const cursor = page.nextCursor; + page = yield* abortablePromise( + (signal) => + client.listTools({ + params: { cursor }, + options: { signal }, + }), + () => catalogFailed(caseId), + ); + tools.push(...page.tools); + pageCount += 1; + } + return { ...page, tools }; + }); + +const acquireMcpClient = ( + caseId: string, + options: ValidatedOmpHarnessTrialOptions, +): Effect.Effect => + abortablePromise( + (signal) => + createMCPClient({ + transport: { + type: "http", + url: options.serverUrl, + headers: { Authorization: `Bearer ${options.mcpAuthorization}` }, + redirect: "error", + }, + initializationOptions: { signal }, + maxRetries: 0, + clientName: "ask-gina-omp-eval", + }).then((client) => { + if (!signal.aborted) return client; + void closeMcpClientOnce(client); + return Promise.reject(signal.reason); + }), + () => new PluginEvalOmpHarnessMcpError({ caseId, reason: "connection-failed" }), + ); + +const releaseMcpClient = ( + client: MCPClient, + exit: Exit.Exit, + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => { + if (Exit.isFailure(exit)) { + return Effect.sync(() => { + void closeMcpClientOnce(client); + }); + } + return Effect.gen(function* () { + const beforeClose = yield* Clock.currentTimeMillis; + if (beforeClose >= deadlineMillis) { + void closeMcpClientOnce(client); + return yield* timeoutError(caseId, timeoutMs); + } + const remainingMs = Math.min( + Math.max(0, Math.trunc(deadlineMillis - beforeClose)), + MAX_MCP_CLOSE_WAIT_MS, + ); + const outcome = yield* Effect.raceFirst( + Effect.promise(() => closeMcpClientOnce(client)), + Effect.sleep(Duration.millis(remainingMs)).pipe(Effect.as("timed-out" as const)), + ); + const afterClose = yield* Clock.currentTimeMillis; + if (afterClose >= deadlineMillis) return yield* timeoutError(caseId, timeoutMs); + if (outcome !== "closed") { + return yield* new PluginEvalOmpHarnessMcpError({ caseId, reason: "cleanup-failed" }); + } + }); +}; + +const releaseHarnessSession = ( + session: HarnessAgentSession, + rawSandboxDestroy: RawSandboxDestroyRef, + exit: Exit.Exit, + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => { + if (Exit.isFailure(exit)) { + return Effect.raceFirst( + Effect.promise(() => destroyHarnessSessionOnce(session, rawSandboxDestroy)), + Effect.sleep(Duration.millis(MAX_SESSION_DESTROY_WAIT_MS)), + ).pipe(Effect.asVoid); + } + return Effect.gen(function* () { + const beforeDestroy = yield* Clock.currentTimeMillis; + if (beforeDestroy >= deadlineMillis) { + void destroyHarnessSessionOnce(session, rawSandboxDestroy); + return yield* timeoutError(caseId, timeoutMs); + } + const remainingMs = Math.min( + Math.max(0, Math.trunc(deadlineMillis - beforeDestroy)), + MAX_SESSION_DESTROY_WAIT_MS, + ); + const outcome = yield* Effect.raceFirst( + Effect.promise(() => destroyHarnessSessionOnce(session, rawSandboxDestroy)), + Effect.sleep(Duration.millis(remainingMs)).pipe(Effect.as("timed-out" as const)), + ); + const afterDestroy = yield* Clock.currentTimeMillis; + if (afterDestroy >= deadlineMillis) return yield* timeoutError(caseId, timeoutMs); + if (outcome !== "destroyed") { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId, + reason: "generation-failed", + }); + } + }); +}; + +const loadStagedSkills = ( + runtimeDirectory: string, + caseId: string, +): Effect.Effect< + ReadonlyArray<{ readonly name: string; readonly description: string; readonly content: string }>, + PluginEvalOmpHarnessRequestError, + FileSystem.FileSystem | Path.Path +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return yield* Effect.forEach(SKILL_NAMES, (name) => + Effect.gen(function* () { + const skillPath = path.join(runtimeDirectory, "skills", name, "SKILL.md"); + const content = yield* fs.readFileString(skillPath); + const parsed = yield* parseSkillFrontmatter(content, name); + return { + name, + description: parsed.description, + content: content.replace(SKILL_FRONTMATTER, "").trimStart(), + }; + }), + ); + }).pipe( + Effect.mapError( + () => new PluginEvalOmpHarnessRequestError({ caseId, reason: "invalid-options" }), + ), + ); + +const createInputSchemaCompiler = (): ((schema: unknown) => ValidateFunction | undefined) => { + const ajv = addFormats( + new Ajv({ + allErrors: false, + coerceTypes: false, + useDefaults: false, + removeAdditional: false, + strict: true, + validateFormats: true, + }), + ); + return (schema: unknown): ValidateFunction | undefined => { + if (typeof schema !== "object" || schema === null || Array.isArray(schema)) return undefined; + if (Object.hasOwn(schema, "$async")) return undefined; + try { + return ajv.compile(schema); + } catch { + return undefined; + } + }; +}; + +const decodeEvidenceText = ( + evidenceText: string | null | undefined, +): OmpGuardEvidence | undefined => { + if (evidenceText === null || evidenceText === undefined) return undefined; + try { + return decodeOmpGuardEvidence(JSON.parse(evidenceText) as unknown); + } catch { + return undefined; + } +}; + +const executeCanonicalHostTool = ( + run: () => PromiseLike, + validator: ValidateFunction, + input: unknown, + executeOptions: { readonly toolCallId: string; readonly abortSignal?: AbortSignal }, + canonicalName: string, + captures: CapturedHostToolCall[], + trustedSession: { current: EvidenceSandbox | undefined }, +): Promise => + Effect.runPromise( + Effect.gen(function* () { + const argumentsValue = jsonObject(input); + const pushCapture = (extra: Pick): void => { + captures.push({ + toolCallId: executeOptions.toolCallId, + name: canonicalName, + arguments: argumentsValue ?? {}, + ...extra, + }); + }; + if (argumentsValue === undefined || validator(input) !== true) { + pushCapture({ + error: { code: "invalid_arguments", message: "Invalid tool arguments" }, + }); + return { isError: true }; + } + const sandbox = trustedSession.current; + if (sandbox === undefined) { + pushCapture({ error: { message: "OMP harness inventory was not admitted" } }); + return { isError: true }; + } + const evidenceText = yield* Effect.promise(() => + Promise.resolve(sandbox.readTextFile({ path: CONTAINER_EVIDENCE_PATH })).then( + (text) => text, + () => null, + ), + ); + const evidence = decodeEvidenceText(evidenceText); + if (!inventoryAdmitted(evidence)) { + pushCapture({ error: { message: "OMP harness inventory was not admitted" } }); + return { isError: true }; + } + const output = yield* Effect.promise(() => + Promise.resolve(run()).then( + (value) => value, + () => undefined, + ), + ); + if (output === undefined) { + pushCapture({ error: { message: "MCP tool call failed" } }); + return { isError: true }; + } + if ( + typeof output === "object" && + output !== null && + !Array.isArray(output) && + "isError" in output && + output.isError === true + ) { + pushCapture({ + resultBytes: resultByteLength(output), + error: { message: "MCP tool call failed" }, + }); + return output; + } + pushCapture({ resultBytes: resultByteLength(output) }); + return output; + }), + executeOptions.abortSignal === undefined ? undefined : { signal: executeOptions.abortSignal }, + ); + +const preflightOmpHarnessSession = ( + session: EvidenceSandbox, + trustedSession: { current: EvidenceSandbox | undefined }, + caseId: string, + abortSignal: AbortSignal | undefined, +): Promise => + Effect.runPromise( + Effect.gen(function* () { + trustedSession.current = session; + const version = yield* Effect.tryPromise({ + try: (signal) => + session.run({ + command: `${CONTAINER_OMP_PATH} --version`, + abortSignal: abortSignal ?? signal, + }), + catch: () => + new PluginEvalOmpHarnessSpawnError({ + caseId, + reason: "preflight-failed", + }), + }); + if (version.exitCode !== 0) { + return yield* new PluginEvalOmpHarnessSpawnError({ + caseId, + reason: "preflight-failed", + }); + } + if (version.stdout.trim() !== `omp/${OMP_REQUIRED_VERSION}`) { + return yield* new PluginEvalOmpHarnessSpawnError({ + caseId, + reason: "unsupported-version", + }); + } + }), + abortSignal === undefined ? undefined : { signal: abortSignal }, + ); + +const wrapHostTools = ( + tools: ToolSet, + definitions: ListToolsResult, + discoveredTools: readonly string[], + captures: CapturedHostToolCall[], + trustedSession: { current: EvidenceSandbox | undefined }, + caseId: string, +): Effect.Effect => + Effect.gen(function* () { + const compileInputSchema = createInputSchemaCompiler(); + const definitionsByName = new Map(definitions.tools.map((tool) => [tool.name, tool] as const)); + const hostTools: ToolSet = {}; + for (const canonicalName of discoveredTools) { + const tool = tools[canonicalName]; + const definition = definitionsByName.get(canonicalName); + const execute = tool?.execute; + if ( + tool === undefined || + execute === undefined || + definition === undefined || + !NATIVE_TOOL_NAME.test(canonicalName.replaceAll(".", "_")) + ) { + return yield* new PluginEvalOmpHarnessMcpError({ caseId, reason: "catalog-mismatch" }); + } + const validator = compileInputSchema(definition.inputSchema); + if (validator === undefined) { + return yield* catalogFailed(caseId); + } + hostTools[canonicalName] = { + ...tool, + execute: (input, executeOptions) => + executeCanonicalHostTool( + () => execute(input, executeOptions), + validator, + input, + executeOptions, + canonicalName, + captures, + trustedSession, + ), + }; + } + return hostTools; + }); + +const observedToolCalls = ( + steps: readonly StepResult[], + captures: readonly CapturedHostToolCall[], + nativeCalls: OmpGuardEvidence["nativeCalls"], +): ObservedOmpToolCalls | undefined => { + const capturesById = new Map(captures.map((capture) => [capture.toolCallId, capture] as const)); + if (capturesById.size !== captures.length) return undefined; + + const hostCallsByName = new Map(); + const seenHostCallIds = new Set(); + for (const step of steps) { + for (const toolCall of step.toolCalls) { + if (CANONICAL_TOOL_NAMES[toolCall.toolName] !== true) continue; + if (toolCall.providerExecuted !== false || seenHostCallIds.has(toolCall.toolCallId)) { + return undefined; + } + seenHostCallIds.add(toolCall.toolCallId); + const captured = capturesById.get(toolCall.toolCallId); + const argumentsValue = jsonObject(toolCall.input) ?? captured?.arguments ?? {}; + let observed: PluginEvalToolCall; + if (toolCall.invalid === true && captured === undefined) { + observed = { + sequence: 0, + name: toolCall.toolName, + arguments: argumentsValue, + error: { code: "invalid_arguments", message: "Invalid tool arguments" }, + }; + } else { + if (captured === undefined || captured.name !== toolCall.toolName) return undefined; + observed = { + sequence: 0, + name: captured.name, + arguments: captured.arguments, + ...(captured.durationMs === undefined ? {} : { duration_ms: captured.durationMs }), + ...(captured.resultBytes === undefined ? {} : { result_bytes: captured.resultBytes }), + ...(captured.error === undefined ? {} : { error: captured.error }), + }; + } + const namedCalls = hostCallsByName.get(observed.name); + if (namedCalls === undefined) hostCallsByName.set(observed.name, [observed]); + else namedCalls.push(observed); + } + } + for (const capture of captures) { + if (!seenHostCallIds.has(capture.toolCallId)) return undefined; + } + + const nativeCallById = new Map(nativeCalls.map((call) => [call.id, call] as const)); + if (nativeCallById.size !== nativeCalls.length) return undefined; + const sdkNativeCalls = new Map(); + for (const step of steps) { + for (const part of step.content) { + if (part.type === "tool-call") { + if (CANONICAL_TOOL_NAMES[part.toolName] === true) continue; + if ( + part.providerExecuted !== true || + !nativeCallById.has(part.toolCallId) || + sdkNativeCalls.has(part.toolCallId) + ) { + return undefined; + } + sdkNativeCalls.set(part.toolCallId, { input: part.input }); + continue; + } + if (part.type !== "tool-result" && part.type !== "tool-error") continue; + if (CANONICAL_TOOL_NAMES[part.toolName] === true) continue; + const sdkCall = sdkNativeCalls.get(part.toolCallId); + if ( + !nativeCallById.has(part.toolCallId) || + sdkCall === undefined || + sdkCall.outcome !== undefined + ) { + return undefined; + } + sdkCall.outcome = part.type === "tool-error" ? "error" : "result"; + } + } + + const toolCalls: PluginEvalToolCall[] = []; + let failedNativeRead = false; + for (const nativeCall of nativeCalls) { + const sdkCall = sdkNativeCalls.get(nativeCall.id); + if (nativeCall.name === "read") { + if (sdkCall?.outcome === undefined) return undefined; + if (sdkCall.outcome === "error") failedNativeRead = true; + continue; + } + + const canonicalName = Object.hasOwn(CANONICAL_TOOL_BY_NATIVE_NAME, nativeCall.name) + ? CANONICAL_TOOL_BY_NATIVE_NAME[nativeCall.name] + : undefined; + if (canonicalName === undefined) { + if (sdkCall?.outcome !== "error") return undefined; + continue; + } + if (sdkCall !== undefined) { + if (sdkCall.outcome !== "error") return undefined; + const argumentsValue = jsonObject(sdkCall.input); + if (argumentsValue === undefined) return undefined; + toolCalls.push({ + sequence: toolCalls.length, + name: canonicalName, + arguments: argumentsValue, + error: { message: "MCP tool call failed" }, + }); + continue; + } + + const hostCalls = hostCallsByName.get(canonicalName); + const hostCall = hostCalls?.shift(); + if (hostCall === undefined) return undefined; + toolCalls.push({ ...hostCall, sequence: toolCalls.length }); + } + for (const hostCalls of hostCallsByName.values()) { + if (hostCalls.length > 0) return undefined; + } + return { toolCalls, failedNativeRead }; +}; + +const promptText = (evalCase: PluginEvalCase): string => + evalCase.turns + .filter((turn) => turn.role === "user") + .map((turn) => turn.content) + .join("\n\n"); + +const createHarnessSession = ( + agent: HarnessAgent, + rawSandboxDestroy: RawSandboxDestroyRef, + caseId: string, +): Effect.Effect => + Effect.callback((resume, signal) => { + void Promise.resolve() + .then(() => agent.createSession({ abortSignal: signal })) + .then( + (session) => { + if (!signal.aborted) { + resume(Effect.succeed(session)); + return; + } + void destroyHarnessSessionOnce(session, rawSandboxDestroy); + }, + (error) => { + if (rawSandboxDestroy.current !== undefined) { + void settleHarnessDestroy(rawSandboxDestroy.current); + } + if (signal.aborted) return; + resume( + Effect.fail( + isKnownHarnessError(error) + ? error + : new PluginEvalOmpHarnessSpawnError({ + caseId, + reason: "could_not_start", + }), + ), + ); + }, + ); + }); + +export const runOmpHarnessPluginEvalTrial = Function.dual< + ( + options: OmpHarnessTrialOptions, + ) => ( + evalCase: PluginEvalCase, + ) => Effect.Effect< + PluginEvalObservation, + PluginEvalOmpHarnessError, + FileSystem.FileSystem | Path.Path + >, + ( + evalCase: PluginEvalCase, + options: OmpHarnessTrialOptions, + ) => Effect.Effect< + PluginEvalObservation, + PluginEvalOmpHarnessError, + FileSystem.FileSystem | Path.Path + > +>( + 2, + ( + evalCase: PluginEvalCase, + options: OmpHarnessTrialOptions, + ): Effect.Effect< + PluginEvalObservation, + PluginEvalOmpHarnessError, + FileSystem.FileSystem | Path.Path + > => + Effect.gen(function* () { + const validated = yield* validateOptions(evalCase, options); + const startedMillis = yield* Clock.currentTimeMillis; + const startedAt = DateTime.formatIso(DateTime.makeUnsafe(startedMillis)); + const deadlineMillis = startedMillis + validated.timeoutMs; + + return yield* Effect.gen(function* () { + const skills = yield* withRunDeadline( + loadStagedSkills(validated.runtimeDirectory, evalCase.id), + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ); + const trialResult = yield* withRunDeadline( + Effect.uninterruptibleMask((restore) => + Effect.flatMap(restore(acquireMcpClient(evalCase.id, validated)), (client) => + restore( + Effect.gen(function* () { + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const definitions = yield* listAllMcpTools(client, evalCase.id); + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const discoveredTools = definitions.tools.map(({ name }) => name); + if (!catalogsMatch(discoveredTools, validated.availableTools)) { + return yield* new PluginEvalOmpHarnessMcpError({ + caseId: evalCase.id, + reason: "catalog-mismatch", + }); + } + const allowed = new Set(validated.availableTools); + const tools = yield* Effect.try({ + try: () => + client.toolsFromDefinitions({ + ...definitions, + tools: definitions.tools.filter(({ name }) => allowed.has(name)), + }), + catch: () => + new PluginEvalOmpHarnessMcpError({ + caseId: evalCase.id, + reason: "catalog-failed", + }), + }); + const captures: CapturedHostToolCall[] = []; + const trustedSession: { current: EvidenceSandbox | undefined } = { + current: undefined, + }; + const hostTools = yield* wrapHostTools( + tools, + definitions, + discoveredTools, + captures, + trustedSession, + evalCase.id, + ); + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const selectedSandbox = + validated.sandbox ?? + createOmpDockerSandbox({ + runtimeDirectory: validated.runtimeDirectory, + ...(validated.dockerImage === undefined + ? {} + : { image: validated.dockerImage }), + ...(validated.dockerNetwork === undefined + ? {} + : { network: validated.dockerNetwork }), + }); + const rawSandboxDestroy: RawSandboxDestroyRef = { current: undefined }; + const sandbox: HarnessV1SandboxProvider = { + ...selectedSandbox, + createSession: (sessionOptions) => + Promise.resolve(selectedSandbox.createSession(sessionOptions)).then( + (session) => { + if (typeof session.destroy !== "function") { + throw new Error("OMP sandbox session cleanup unavailable"); + } + rawSandboxDestroy.current = () => session.destroy(); + return session; + }, + ), + }; + const harness = createACP({ + harnessId: "omp-acp", + builtinTools: OMP_NATIVE_TOOL_METADATA, + source: { + type: "install-command", + command: installCommand( + validated.provider, + validated.model, + validated.providerBaseUrl, + ), + }, + executable: "omp", + args: [ + "acp", + "--trusted-extension", + CONTAINER_GUARD_PATH, + "--no-extensions", + "--provider", + OMP_EVAL_PROVIDER_ALIAS, + "--model", + validated.model, + "--thinking", + validated.reasoning, + "--approval-mode", + "yolo", + "--no-session", + ], + skillsDirectory: ".omp/agent/skills", + modelMapping: { type: "session-config-option", path: "model" }, + mcpServers: {}, + credentialEnv: [OMP_EVAL_PROVIDER_API_KEY_ENV], + credentialBrokering: ({ env, sandboxEnv }) => { + const key = env[OMP_EVAL_PROVIDER_API_KEY_ENV]; + const placeholder = sandboxEnv?.[OMP_EVAL_PROVIDER_API_KEY_ENV]; + if (key === undefined || placeholder === undefined) { + throw new Error("OMP provider credentials unavailable"); + } + const matchUrl = + validated.providerBaseUrl ?? PROVIDER_PROFILE[validated.provider].baseUrl; + const officialAnthropic = + validated.provider === "anthropic" && + new URL(matchUrl).origin === PROVIDER_PROFILE.anthropic.baseUrl; + const header = officialAnthropic ? "x-api-key" : "authorization"; + const prefix = officialAnthropic ? "" : "Bearer "; + return [ + createCredentialRequestTransformation({ + matchUrl, + matchHeaders: { [header]: `${prefix}${placeholder}` }, + transformHeaders: { [header]: `${prefix}${key}` }, + }), + ]; + }, + auth: { [OMP_EVAL_PROVIDER_API_KEY_ENV]: validated.apiKey }, + env: { NO_COLOR: "1" }, + }); + const agent = new HarnessAgent({ + harness, + sandbox, + tools: hostTools, + skills, + permissionMode: "allow-all", + sandboxConfig: { + onSession: ({ session, abortSignal }) => + preflightOmpHarnessSession( + session, + trustedSession, + evalCase.id, + abortSignal, + ), + }, + }); + return yield* withRunDeadline( + Effect.uninterruptibleMask((restoreSession) => + Effect.flatMap( + restoreSession(createHarnessSession(agent, rawSandboxDestroy, evalCase.id)), + (session) => + restoreSession( + Effect.gen(function* () { + yield* ensureBeforeDeadline( + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ); + const generatedText = yield* Effect.tryPromise({ + try: (signal) => + agent.generate({ + session, + prompt: promptText(evalCase), + abortSignal: signal, + }), + catch: (error) => + isKnownHarnessError(error) + ? error + : new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "generation-failed", + }), + }); + yield* ensureBeforeDeadline( + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ); + const evidenceSandbox = trustedSession.current; + if (evidenceSandbox === undefined) { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "incomplete-evidence", + }); + } + const evidenceText = yield* Effect.promise(() => + Promise.resolve( + evidenceSandbox.readTextFile({ + path: CONTAINER_EVIDENCE_PATH, + }), + ).then( + (text) => text, + () => null, + ), + ); + const evidence = decodeEvidenceText(evidenceText); + if (evidence === undefined) { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "incomplete-evidence", + }); + } + return { generatedText, evidence, captures }; + }), + ).pipe( + Effect.onExit((exit) => + releaseHarnessSession( + session, + rawSandboxDestroy, + exit, + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ), + ), + ), + ), + ), + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ); + }), + ).pipe( + Effect.onExit((exit) => + releaseMcpClient(client, exit, evalCase.id, validated.timeoutMs, deadlineMillis), + ), + ), + ), + ), + evalCase.id, + validated.timeoutMs, + deadlineMillis, + ); + + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const { generatedText, evidence, captures } = trialResult; + if (evidence.phase !== "terminal" || evidence.terminal === undefined) { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "incomplete-evidence", + }); + } + if (!inventoryAdmitted(evidence)) { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "inventory-mismatch", + }); + } + const observed = observedToolCalls(generatedText.steps, captures, evidence.nativeCalls); + if (observed === undefined) { + return yield* new PluginEvalOmpHarnessProcessError({ + caseId: evalCase.id, + reason: "incomplete-evidence", + }); + } + const { toolCalls, failedNativeRead } = observed; + const terminal = evidence.terminal; + const tokenUsage = + terminal.usage === undefined + ? undefined + : { + input_tokens: terminal.usage.inputTokens, + output_tokens: terminal.usage.outputTokens, + total_tokens: terminal.usage.totalTokens, + }; + const incompleteStop = + terminal.stopReason === "length" || + terminal.stopReason === "toolUse" || + terminal.stopReason === "error" || + terminal.stopReason === "aborted" || + terminal.isError; + const blocked = evidence.blockedActions > 0; + const failedTool = toolCalls.some((call) => call.error !== undefined); + const completed = + terminal.stopReason === "stop" && + !incompleteStop && + !blocked && + !failedTool && + !failedNativeRead; + const finishedMillis = yield* Clock.currentTimeMillis; + if (finishedMillis >= deadlineMillis) { + return yield* timeoutError(evalCase.id, validated.timeoutMs); + } + const error = completed + ? undefined + : blocked + ? OMP_POLICY_ERROR + : failedTool || failedNativeRead + ? OMP_TOOL_EXECUTION_ERROR + : OMP_INCOMPLETE_GENERATION_ERROR; + return { + version: 1, + run_id: validated.runId, + case_id: evalCase.id, + target: "omp_harness", + model: validated.modelIdentity, + repetition: validated.repetition, + started_at: startedAt, + status: error === undefined ? "completed" : "failed", + duration_ms: Math.max(0, finishedMillis - startedMillis), + activated_skills: [...evidence.activatedSkills], + tool_calls: [...toolCalls], + available_tools: [...validated.availableTools], + ...(tokenUsage === undefined ? {} : { token_usage: tokenUsage }), + ...(generatedText.text.length === 0 ? {} : { final_answer: generatedText.text }), + ...(error === undefined ? {} : { error }), + } satisfies PluginEvalObservation; + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(validated.timeoutMs), + orElse: () => Effect.fail(timeoutError(evalCase.id, validated.timeoutMs)), + }), + ); + }).pipe( + Effect.withSpan("plugin_evals.omp_harness_trial", { + attributes: { + "plugin_eval.case_id": evalCase.id, + "plugin_eval.repetition": options.repetition, + }, + }), + ), +); diff --git a/packages/evals/src/openrouter.ts b/packages/evals/src/openrouter.ts new file mode 100644 index 0000000..662b85d --- /dev/null +++ b/packages/evals/src/openrouter.ts @@ -0,0 +1,598 @@ +import { createMCPClient, type ListToolsResult, type MCPClient } from "@ai-sdk/mcp"; +import { listCatalogToolNames, PRODUCTION_MCP_URL } from "@askgina/contracts"; +import { createOpenRouter } from "@openrouter/ai-sdk-provider"; +import { generateText, isStepCount, type StepResult, type ToolSet } from "ai"; +import { Clock, Data, DateTime, Duration, Effect, Exit, Function } from "effect"; + +import type { + PluginEvalCase, + PluginEvalObservation, + PluginEvalTokenUsage, + PluginEvalToolCall, +} from "./contracts"; + +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_MAX_STEPS = 8; +const MAX_MAX_STEPS = 32; +const MAX_MCP_TOOL_PAGES = 32; +const MAX_MCP_CLOSE_WAIT_MS = 1_000; +const INCOMPLETE_GENERATION_ERROR = "OpenRouter generation did not complete with a final answer"; +const UTF8_ENCODER = new TextEncoder(); +const CANONICAL_ALLOWED_TOOLS = listCatalogToolNames(); +const OPENROUTER_WIRE_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/; + +const OPENROUTER_REASONING_EFFORTS = { + none: true, + minimal: true, + low: true, + medium: true, + high: true, + xhigh: true, +} as const; + +type OpenRouterReasoningEffort = keyof typeof OPENROUTER_REASONING_EFFORTS; + +export interface OpenRouterTrialOptions { + readonly apiKey: string; + readonly mcpAuthorization: string; + readonly model: string; + readonly reasoning: string; + readonly runId: string; + readonly repetition: number; + readonly serverUrl: string; + readonly allowedTools: readonly string[]; + readonly timeoutMs?: number; + readonly maxSteps?: number; +} + +interface ValidatedOpenRouterTrialOptions { + readonly apiKey: string; + readonly mcpAuthorization: string; + readonly model: string; + readonly reasoning: OpenRouterReasoningEffort; + readonly runId: string; + readonly repetition: number; + readonly serverUrl: string; + readonly allowedTools: readonly string[]; + readonly timeoutMs: number; + readonly maxSteps: number; +} + +export class PluginEvalOpenRouterRequestError extends Data.TaggedError( + "PluginEvalOpenRouterRequestError", +)<{ + readonly caseId: string; + readonly reason: "invalid-options" | "unsupported-reasoning"; +}> {} + +export class PluginEvalOpenRouterMcpError extends Data.TaggedError("PluginEvalOpenRouterMcpError")<{ + readonly caseId: string; + readonly reason: "connection-failed" | "catalog-failed" | "catalog-mismatch" | "cleanup-failed"; +}> {} + +export class PluginEvalOpenRouterGenerationError extends Data.TaggedError( + "PluginEvalOpenRouterGenerationError", +)<{ + readonly caseId: string; + readonly reason: "generation-failed"; +}> {} + +export class PluginEvalOpenRouterTimeoutError extends Data.TaggedError( + "PluginEvalOpenRouterTimeoutError", +)<{ + readonly caseId: string; + readonly timeoutMs: number; +}> {} + +export type PluginEvalOpenRouterError = + | PluginEvalOpenRouterRequestError + | PluginEvalOpenRouterMcpError + | PluginEvalOpenRouterGenerationError + | PluginEvalOpenRouterTimeoutError; + +const catalogsMatch = (left: readonly string[], right: readonly string[]): boolean => { + if (left.length !== right.length) return false; + const uniqueLeft = new Set(left); + return uniqueLeft.size === left.length && right.every((tool) => uniqueLeft.has(tool)); +}; + +const isReasoningEffort = (value: string): value is OpenRouterReasoningEffort => + Object.hasOwn(OPENROUTER_REASONING_EFFORTS, value); + +const validateOptions = ( + evalCase: PluginEvalCase, + options: OpenRouterTrialOptions, +): Effect.Effect => { + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS; + const commonOptionsAreValid = + options.serverUrl === PRODUCTION_MCP_URL && + options.apiKey.trim().length > 0 && + options.mcpAuthorization.trim().length > 0 && + options.model.trim().length > 0 && + options.runId.trim().length > 0 && + Number.isSafeInteger(options.repetition) && + options.repetition > 0 && + Number.isSafeInteger(timeoutMs) && + timeoutMs > 0 && + Number.isSafeInteger(maxSteps) && + maxSteps > 0 && + maxSteps <= MAX_MAX_STEPS && + catalogsMatch(options.allowedTools, CANONICAL_ALLOWED_TOOLS); + + if (!commonOptionsAreValid) { + return Effect.fail( + new PluginEvalOpenRouterRequestError({ + caseId: evalCase.id, + reason: "invalid-options", + }), + ); + } + if (!isReasoningEffort(options.reasoning)) { + return Effect.fail( + new PluginEvalOpenRouterRequestError({ + caseId: evalCase.id, + reason: "unsupported-reasoning", + }), + ); + } + + return Effect.succeed({ + apiKey: options.apiKey, + mcpAuthorization: options.mcpAuthorization, + model: options.model, + reasoning: options.reasoning, + runId: options.runId, + repetition: options.repetition, + serverUrl: options.serverUrl, + allowedTools: [...options.allowedTools], + timeoutMs, + maxSteps, + }); +}; + +const catalogFailed = (caseId: string): PluginEvalOpenRouterMcpError => + new PluginEvalOpenRouterMcpError({ + caseId, + reason: "catalog-failed", + }); + +const abortablePromise = ( + run: (signal: AbortSignal) => PromiseLike, + onError: () => E, +): Effect.Effect => + Effect.callback((resume, signal) => { + void Promise.resolve() + .then(() => run(signal)) + .then( + (value) => { + if (!signal.aborted) resume(Effect.succeed(value)); + }, + () => { + if (!signal.aborted) resume(Effect.fail(onError())); + }, + ); + }); + +const listAllMcpTools = ( + client: MCPClient, + caseId: string, +): Effect.Effect => + Effect.gen(function* () { + let page = yield* abortablePromise( + (signal) => client.listTools({ options: { signal } }), + () => catalogFailed(caseId), + ); + const tools = [...page.tools]; + const seenCursors = new Set(); + let pageCount = 1; + + while (page.nextCursor !== undefined) { + if (pageCount >= MAX_MCP_TOOL_PAGES || seenCursors.has(page.nextCursor)) { + return yield* catalogFailed(caseId); + } + seenCursors.add(page.nextCursor); + const cursor = page.nextCursor; + page = yield* abortablePromise( + (signal) => + client.listTools({ + params: { cursor }, + options: { signal }, + }), + () => catalogFailed(caseId), + ); + tools.push(...page.tools); + pageCount += 1; + } + + return { ...page, tools }; + }); + +const isJsonValue = (value: unknown): boolean => { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) + ) { + return true; + } + if (Array.isArray(value)) return value.every(isJsonValue); + if (typeof value !== "object") return false; + return Object.values(value).every(isJsonValue); +}; + +const jsonObject = (value: unknown): PluginEvalToolCall["arguments"] | undefined => { + if (typeof value !== "object" || value === null || Array.isArray(value) || !isJsonValue(value)) { + return undefined; + } + return value as PluginEvalToolCall["arguments"]; +}; + +const nonNegativeInteger = (value: unknown): number | undefined => + typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +const completeTokenUsage = (usage: { + readonly inputTokens: number | undefined; + readonly outputTokens: number | undefined; + readonly totalTokens: number | undefined; +}): PluginEvalTokenUsage | undefined => { + const inputTokens = nonNegativeInteger(usage.inputTokens); + const outputTokens = nonNegativeInteger(usage.outputTokens); + const totalTokens = nonNegativeInteger(usage.totalTokens); + return inputTokens === undefined || outputTokens === undefined || totalTokens === undefined + ? undefined + : { + input_tokens: inputTokens, + output_tokens: outputTokens, + total_tokens: totalTokens, + }; +}; + +const resultByteLength = (value: unknown): number | undefined => { + try { + const serialized = typeof value === "string" ? value : JSON.stringify(value); + return serialized === undefined ? undefined : UTF8_ENCODER.encode(serialized).byteLength; + } catch { + return undefined; + } +}; + +const observedToolCalls = ( + steps: readonly StepResult[], + wireToCanonical: ReadonlyMap, +): readonly PluginEvalToolCall[] => { + const observed: PluginEvalToolCall[] = []; + + for (const step of steps) { + for (const toolCall of step.toolCalls) { + const canonicalName = wireToCanonical.get(toolCall.toolName); + const argumentsValue = jsonObject(toolCall.input); + const duration = nonNegativeInteger(step.performance.toolExecutionMs[toolCall.toolCallId]); + let resultBytes: number | undefined; + let resultObserved = false; + let callError: PluginEvalToolCall["error"] = + canonicalName === undefined || toolCall.invalid === true + ? { + code: "invalid_tool", + message: "OpenRouter requested an unavailable MCP tool", + } + : argumentsValue === undefined + ? { + code: "invalid_arguments", + message: "OpenRouter returned invalid MCP arguments", + } + : undefined; + + for (const part of step.content) { + if ( + (part.type !== "tool-result" && part.type !== "tool-error") || + part.toolCallId !== toolCall.toolCallId + ) { + continue; + } + resultObserved = true; + if (part.type === "tool-error") { + callError ??= { message: "MCP tool call failed" }; + } else { + resultBytes = resultByteLength(part.output); + if ( + typeof part.output === "object" && + part.output !== null && + !Array.isArray(part.output) && + Reflect.get(part.output, "isError") === true + ) { + callError ??= { message: "MCP tool call failed" }; + } + } + break; + } + + if (!resultObserved && canonicalName !== undefined && toolCall.invalid !== true) { + callError ??= { + code: "missing_result", + message: "MCP tool result was not observed", + }; + } + + observed.push({ + sequence: observed.length, + name: canonicalName ?? toolCall.toolName, + arguments: argumentsValue ?? {}, + ...(duration === undefined ? {} : { duration_ms: duration }), + ...(resultBytes === undefined ? {} : { result_bytes: resultBytes }), + ...(callError === undefined ? {} : { error: callError }), + }); + } + } + + return observed; +}; + +const timeoutError = (caseId: string, timeoutMs: number): PluginEvalOpenRouterTimeoutError => + new PluginEvalOpenRouterTimeoutError({ caseId, timeoutMs }); + +const ensureBeforeDeadline = ( + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => + Effect.flatMap(Clock.currentTimeMillis, (now) => + now >= deadlineMillis ? Effect.fail(timeoutError(caseId, timeoutMs)) : Effect.void, + ); + +type McpCloseOutcome = "closed" | "failed"; + +const mcpClientCloses = new WeakMap>(); + +const closeMcpClientOnce = (client: MCPClient): Promise => { + const activeClose = mcpClientCloses.get(client); + if (activeClose !== undefined) return activeClose; + + let closeResult: PromiseLike; + try { + closeResult = client.close(); + } catch { + const failed = Promise.resolve("failed"); + mcpClientCloses.set(client, failed); + return failed; + } + + const outcome = Promise.resolve(closeResult).then( + (): McpCloseOutcome => "closed", + (): McpCloseOutcome => "failed", + ); + mcpClientCloses.set(client, outcome); + return outcome; +}; + +const acquireMcpClient = ( + caseId: string, + options: ValidatedOpenRouterTrialOptions, +): Effect.Effect => + abortablePromise( + (signal) => + createMCPClient({ + transport: { + type: "http", + url: options.serverUrl, + headers: { Authorization: `Bearer ${options.mcpAuthorization}` }, + redirect: "error", + }, + initializationOptions: { signal }, + maxRetries: 0, + clientName: "ask-gina-openrouter-eval", + }).then((client) => { + if (!signal.aborted) return client; + void closeMcpClientOnce(client); + return Promise.reject(signal.reason); + }), + () => + new PluginEvalOpenRouterMcpError({ + caseId, + reason: "connection-failed", + }), + ); + +const releaseMcpClient = ( + client: MCPClient, + exit: Exit.Exit, + caseId: string, + timeoutMs: number, + deadlineMillis: number, +): Effect.Effect => { + if (Exit.isFailure(exit)) { + return Effect.sync(() => { + void closeMcpClientOnce(client); + }); + } + + return Effect.gen(function* () { + const beforeClose = yield* Clock.currentTimeMillis; + if (beforeClose >= deadlineMillis) { + void closeMcpClientOnce(client); + return yield* timeoutError(caseId, timeoutMs); + } + + const remainingMs = Math.min( + Math.max(0, Math.trunc(deadlineMillis - beforeClose)), + MAX_MCP_CLOSE_WAIT_MS, + ); + const outcome = yield* Effect.raceFirst( + Effect.promise(() => closeMcpClientOnce(client)), + Effect.sleep(Duration.millis(remainingMs)).pipe(Effect.as("timed-out" as const)), + ); + const afterClose = yield* Clock.currentTimeMillis; + if (afterClose >= deadlineMillis) return yield* timeoutError(caseId, timeoutMs); + if (outcome !== "closed") { + return yield* new PluginEvalOpenRouterMcpError({ + caseId, + reason: "cleanup-failed", + }); + } + }); +}; + +export const runOpenRouterPluginEvalTrial = Function.dual< + ( + options: OpenRouterTrialOptions, + ) => ( + evalCase: PluginEvalCase, + ) => Effect.Effect, + ( + evalCase: PluginEvalCase, + options: OpenRouterTrialOptions, + ) => Effect.Effect +>(2, (evalCase, options) => + Effect.gen(function* () { + const validated = yield* validateOptions(evalCase, options); + const startedMillis = yield* Clock.currentTimeMillis; + const startedAt = DateTime.formatIso(DateTime.makeUnsafe(startedMillis)); + const deadlineMillis = startedMillis + validated.timeoutMs; + + return yield* Effect.gen(function* () { + const generated = yield* Effect.uninterruptibleMask((restore) => + Effect.flatMap(restore(acquireMcpClient(evalCase.id, validated)), (client) => + restore( + Effect.gen(function* () { + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const definitions = yield* listAllMcpTools(client, evalCase.id); + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + const discoveredTools = definitions.tools.map(({ name }) => name); + if (!catalogsMatch(discoveredTools, validated.allowedTools)) { + return yield* new PluginEvalOpenRouterMcpError({ + caseId: evalCase.id, + reason: "catalog-mismatch", + }); + } + + const allowed = new Set(validated.allowedTools); + const tools = yield* Effect.try({ + try: () => + client.toolsFromDefinitions({ + ...definitions, + tools: definitions.tools.filter(({ name }) => allowed.has(name)), + }), + catch: () => + new PluginEvalOpenRouterMcpError({ + caseId: evalCase.id, + reason: "catalog-failed", + }), + }); + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + + const wireTools: ToolSet = {}; + const wireToolOrder: string[] = []; + const wireToCanonical = new Map(); + for (const canonicalName of discoveredTools) { + const wireName = canonicalName.replaceAll(".", "_"); + const tool = tools[canonicalName]; + if ( + tool === undefined || + !OPENROUTER_WIRE_TOOL_NAME.test(wireName) || + wireToCanonical.has(wireName) + ) { + return yield* new PluginEvalOpenRouterMcpError({ + caseId: evalCase.id, + reason: "catalog-mismatch", + }); + } + wireTools[wireName] = tool; + wireToolOrder.push(wireName); + wireToCanonical.set(wireName, canonicalName); + } + + const result = yield* abortablePromise( + (signal) => { + const openrouter = createOpenRouter({ + apiKey: validated.apiKey, + compatibility: "strict", + }); + return generateText({ + model: openrouter(validated.model, { usage: { include: true } }), + messages: evalCase.turns.map(({ role, content }) => ({ role, content })), + allowSystemInMessages: true, + tools: wireTools, + toolOrder: wireToolOrder, + toolChoice: "auto", + stopWhen: isStepCount(validated.maxSteps), + maxRetries: 0, + abortSignal: signal, + providerOptions: { + openrouter: { + reasoning: { effort: validated.reasoning }, + }, + }, + }); + }, + () => + new PluginEvalOpenRouterGenerationError({ + caseId: evalCase.id, + reason: "generation-failed", + }), + ); + yield* ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis); + + return { result, discoveredTools, wireToCanonical }; + }), + ).pipe( + Effect.onExit((exit) => + releaseMcpClient(client, exit, evalCase.id, validated.timeoutMs, deadlineMillis), + ), + ), + ), + ).pipe( + Effect.matchEffect({ + onSuccess: (value) => + ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis).pipe( + Effect.as(value), + ), + onFailure: (error) => + ensureBeforeDeadline(evalCase.id, validated.timeoutMs, deadlineMillis).pipe( + Effect.flatMap(() => Effect.fail(error)), + ), + }), + ); + + const finishedMillis = yield* Clock.currentTimeMillis; + if (finishedMillis >= deadlineMillis) { + return yield* timeoutError(evalCase.id, validated.timeoutMs); + } + + const tokenUsage = completeTokenUsage(generated.result.usage); + const finalAnswer = generated.result.text.length === 0 ? undefined : generated.result.text; + const completed = generated.result.finishReason === "stop" && finalAnswer !== undefined; + + return { + version: 1, + run_id: validated.runId, + case_id: evalCase.id, + target: "openrouter_api", + model: validated.model, + repetition: validated.repetition, + started_at: startedAt, + status: completed ? "completed" : "failed", + duration_ms: Math.max(0, Math.trunc(finishedMillis - startedMillis)), + tool_calls: observedToolCalls(generated.result.steps, generated.wireToCanonical), + available_tools: generated.discoveredTools, + ...(tokenUsage === undefined ? {} : { token_usage: tokenUsage }), + ...(finalAnswer === undefined ? {} : { final_answer: finalAnswer }), + ...(completed ? {} : { error: INCOMPLETE_GENERATION_ERROR }), + } satisfies PluginEvalObservation; + }).pipe( + Effect.timeoutOrElse({ + duration: Duration.millis(validated.timeoutMs), + orElse: () => Effect.fail(timeoutError(evalCase.id, validated.timeoutMs)), + }), + ); + }).pipe( + Effect.withSpan("plugin_evals.openrouter_trial", { + attributes: { + "plugin_eval.case_id": evalCase.id, + "plugin_eval.model": options.model, + "plugin_eval.reasoning": options.reasoning, + "plugin_eval.repetition": options.repetition, + }, + }), + ), +); diff --git a/packages/evals/src/public-results.ts b/packages/evals/src/public-results.ts index 7a96d15..863b38b 100644 --- a/packages/evals/src/public-results.ts +++ b/packages/evals/src/public-results.ts @@ -4,6 +4,7 @@ import { PUBLIC_EVAL_DECODE_OPTIONS, PublicEvalAttemptCaptureSchema, PublicEvalIdentifierSchema, + PublicEvalModelSchema, PublicEvalSha256Schema, PublicEvalTimestampSchema, decodePublicEvalResult, @@ -95,7 +96,7 @@ const PositiveIntSchema = Schema.Int.check(Schema.isGreaterThan(0)); const PublicEvalConfigurationSchema = Schema.Struct({ schemaVersion: Schema.Literal("eval-configuration.v1"), candidate: PublicEvalIdentifierSchema, - model: PublicEvalIdentifierSchema, + model: PublicEvalModelSchema, target: PublicEvalIdentifierSchema, reasoning: Schema.NullOr(PublicEvalIdentifierSchema), suiteId: PublicEvalIdentifierSchema, diff --git a/packages/evals/src/replay.ts b/packages/evals/src/replay.ts index 45314bc..9c5cd91 100644 --- a/packages/evals/src/replay.ts +++ b/packages/evals/src/replay.ts @@ -12,6 +12,7 @@ import type { PluginEvalSuite, } from "./contracts"; import { gradePluginEvalObservation, type PluginEvalObservationMismatchError } from "./grading"; +import { validateObservationSetInvariants } from "./load-observations"; import { makePublicEvalAttemptSummaries, assertPublicEvalAttemptPlan, @@ -172,7 +173,11 @@ const validateReplayContract = ( } const requiresCanonicalCatalog = observation.status === "completed" && - (observation.target === "responses_api" || observation.target === "codex_cli"); + (observation.target === "responses_api" || + observation.target === "openrouter_api" || + observation.target === "codex_cli" || + observation.target === "claude_cli" || + observation.target === "omp_harness"); if (requiresCanonicalCatalog && observation.available_tools === undefined) { reasons.push(`${observation.case_id} completed without an imported MCP tool catalog`); } @@ -244,6 +249,9 @@ export const replayPluginEvalObservationSet = Function.dual< args.length >= 2 && typeof args[0] === "object" && args[0] !== null && "cases" in args[0], (suite, observationSet, options) => Effect.gen(function* () { + yield* validateObservationSetInvariants(observationSet, "replay").pipe( + Effect.mapError((error) => new PluginEvalReplayContractError({ reasons: error.reasons })), + ); yield* validateReplayContract(suite, observationSet); const gradedAttempts: PublicEvalGradedAttempt[] | null = options?.captureAttempts === true ? [] : null; diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts index ed60fee..f900f4c 100644 --- a/packages/evals/src/report.ts +++ b/packages/evals/src/report.ts @@ -1,3 +1,4 @@ +import { PublicEvalModelSchema } from "@askgina/contracts"; import { Data, Effect, Schema } from "effect"; import type { PluginEvalReplayReport, PluginEvalRunManifest } from "./contracts"; @@ -32,7 +33,7 @@ export const SanitizedEvalRunReportSchema = Schema.Struct({ runId: SafeRunLabelSchema, candidate: SafeRunLabelSchema, target: SafeRunLabelSchema, - model: SafeRunLabelSchema, + model: PublicEvalModelSchema, reasoning: Schema.optional(SafeRunLabelSchema), repetitions: PositiveIntSchema, startedAt: UtcTimestampSchema, @@ -51,7 +52,7 @@ const SanitizedRunMetadataSchema = Schema.Struct({ runId: SafeRunLabelSchema, candidate: SafeRunLabelSchema, target: SafeRunLabelSchema, - model: SafeRunLabelSchema, + model: PublicEvalModelSchema, reasoning: Schema.optional(SafeRunLabelSchema), accountClass: SafeRunLabelSchema, startedAt: UtcTimestampSchema, diff --git a/tools/pack-artifacts.ts b/tools/pack-artifacts.ts index 96e58c2..534801b 100755 --- a/tools/pack-artifacts.ts +++ b/tools/pack-artifacts.ts @@ -90,6 +90,7 @@ const PACKAGES = [ /^codex-cli-[A-Za-z0-9_-]+\.js\.map$/u, /^index\.d\.ts$/u, /^index\.js$/u, + /^omp-harness-[A-Za-z0-9_-]+\.d\.ts$/u, /^publication-[A-Za-z0-9_-]+\.js$/u, /^publication-[A-Za-z0-9_-]+\.js\.map$/u, /^replay-[A-Za-z0-9_-]+\.js$/u,