diff --git a/README.md b/README.md index b317760..bb63ca7 100644 --- a/README.md +++ b/README.md @@ -66,21 +66,27 @@ requires `ASK_GINA_ACCESS_TOKEN`. Responses and Codex also need `OPENAI_API_KEY` Codex adds `CODEX_EVAL_EXECUTABLE` and `CODEX_EVAL_EXECUTABLE_SHA256`; OpenRouter needs `OPENROUTER_API_KEY` plus required `--openrouter-endpoint`, `--expected-provider`, and `--max-cost-usd`; Claude needs `ANTHROPIC_API_KEY` and -`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_API_KEY`, -`OMP_EVAL_EXECUTABLE`, `OMP_EVAL_EXECUTABLE_SHA256`, and -`--provider openai|anthropic|openrouter`. Optional `--max-steps` applies only to +`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_EXECUTABLE`, +`OMP_EVAL_EXECUTABLE_SHA256`, and `--provider`. Its default API-key mode also +requires `OMP_EVAL_API_KEY` and accepts `openai|anthropic|openrouter`. +`--omp-auth native --omp-agent-dir ` instead uses an +explicit existing OMP profile and its native provider identifier. OMP can +update that profile, including credential refreshes and database writes. +Optional `--max-steps` applies only to OpenRouter and `--max-turns` only to Claude. Both default to 8 and accept 1 to 32. OpenRouter also accepts optional `--server-url` with the exact Gina-read URLs `https://askgina.ai/ai/gina/mcp` (production default) or `https://alpha.askgina.ai/ai/gina/mcp`. Other runners reject those OpenRouter-only -flags. OMP has no step or turn flag. Native Codex, Claude, and OMP paths use -explicit API keys, not a saved personal login. That existing API-key CLI does -not meet a separately requested HarnessAgent or Codex saved-login policy. +flags. OMP has no step or turn flag. Codex and Claude still use explicit API +keys. OMP native mode passed a synthetic stored-credential smoke, not a real +OAuth or subscription authentication test. OpenRouter uses local AI SDK MCP; Responses uses OpenAI-hosted MCP. Neither proves native plugin activation. Codex and Claude adapters distinguish native skill events from task conformance. Claude's live plugin activation remains unverified; offline fixtures and OMP local-runtime proof are not measured native-agent evidence. Stock OMP host-tool success and MCP-error handling passed a cold local synthetic smoke. Native ACP still does -not classify provider HTTP failures. Flags, capture, and publication rules are +not classify provider HTTP failures; this is tracked in +[OMP #11644](https://github.com/can1357/oh-my-pi/issues/11644). +Flags, capture, and publication rules are in `packages/evals/README.md`. diff --git a/packages/evals/README.md b/packages/evals/README.md index fc9957f..9b49495 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -21,6 +21,12 @@ CommonJS, browser and edge runtimes, and subpath imports are unsupported. consumers resolve the same adapter versions as the evaluators. There is no patched adapter, auth-store lease, or restricted-settings surface. +`OmpHarnessTrialOptions` requires a nested `auth: OmpAuth` value. API-key +callers pass `{ mode: "api-key", provider, apiKey }`, optionally with +`providerBaseUrl`; native callers pass `{ mode: "native", provider, agentDirectory }`. The former +top-level `provider`, `apiKey`, and `providerBaseUrl` fields are removed. +The API-key provider type is `OmpApiKeyProvider`. + `createLocalHarnessSandbox` is a synchronous local-process sandbox provider for stock `HarnessAgent`/`createACP` on POSIX hosts. Sessions run commands as ordinary host child processes in their own process group under a disposable @@ -108,24 +114,32 @@ credentials differ: | OpenRouter | `OPENROUTER_API_KEY` | | Codex CLI | `OPENAI_API_KEY`, absolute `CODEX_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `CODEX_EVAL_EXECUTABLE_SHA256` | | Claude CLI | `ANTHROPIC_API_KEY` and absolute `CLAUDE_EVAL_EXECUTABLE` | -| OMP harness | `OMP_EVAL_API_KEY`, absolute `OMP_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256` | - -The supported native Codex, Claude, and OMP paths use explicit API keys in -disposable evaluation homes. They do not reuse a saved personal login. That -existing API-key CLI does not meet a separately requested HarnessAgent or Codex -saved-login policy. Saved-login reuse, refresh ownership, native OAuth -subscriptions, and personal-home integration remain deferred. OMP does -not read `~/.omp` or inherit `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / -`OPENROUTER_API_KEY`. Native OMP also never exports those names into the -child. The child receives only `OMP_EVAL_PROVIDER_API_KEY` for -the private `omp-eval` provider. +| OMP API key | `OMP_EVAL_API_KEY`, absolute `OMP_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256` | +| OMP native | Absolute `OMP_EVAL_EXECUTABLE` and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256` | + +Codex and Claude use explicit API keys in disposable evaluation homes, not a +saved personal login. OMP defaults to `--omp-auth api-key`, which passes the +model key as `OMP_EVAL_PROVIDER_API_KEY` for a private `omp-eval` provider. + +OMP also supports `--omp-auth native --omp-agent-dir `. +The directory must already exist. OMP uses it directly through +`PI_CODING_AGENT_DIR`; the evaluator does not copy credentials, generate native +provider settings, or require `OMP_EVAL_API_KEY`. OMP may refresh credentials, +update its databases, or migrate profile settings there. This is not a +read-only profile mount. Choose the directory deliberately. + +Both OMP modes retain a minimal child environment and disable extensions. +They do not inherit `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or +`OPENROUTER_API_KEY`. Native mode relies on the selected profile's credential +configuration, not arbitrary credential variables in the parent shell. Responses and Codex accept model IDs understood by their OpenAI backends, such as `gpt-5.1`. OpenRouter uses its `provider/model` namespace, such as `openai/gpt-5.1`. Claude uses a Claude CLI model ID or alias without the -`anthropic/` prefix, such as `claude-sonnet-4-5-20250929`. OMP requires `--provider` -to select `openai`, `anthropic`, or `openrouter`; `--model` uses that provider's -backend ID. For example, `--provider openai --model gpt-5.1` records +`anthropic/` prefix, such as `claude-sonnet-4-5-20250929`. OMP API-key mode requires +`--provider openai|anthropic|openrouter`; native mode accepts a native OMP provider +identifier, such as `openai-codex`. `--model` must be available through the +selected provider. For example, `--provider openai --model gpt-5.1` records `openai/gpt-5.1`, while `--provider openrouter --model openai/gpt-5.1` records `openrouter/openai/gpt-5.1`. The report, attempt input, and observation keep that same identity. There is no separate displayed model. These examples show the @@ -189,6 +203,19 @@ bun run eval:omp -- \ --repetitions 3 \ --account-class eval \ --timeout-ms 120000 + +bun run eval:omp -- \ + --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \ + --run-id local-omp-native-example \ + --candidate main \ + --omp-auth native \ + --omp-agent-dir /absolute/path/to/existing/omp-agent \ + --provider openai-codex \ + --model gpt-5.6-luna \ + --reasoning medium \ + --repetitions 3 \ + --account-class eval \ + --timeout-ms 120000 ``` Repeat `--case ` to run a strict subset. `--timeout-ms` is required for @@ -205,8 +232,10 @@ and accepts only the exact Gina-read URLs `https://askgina.ai/ai/gina/mcp` slashes and any other URL, including venue MCP endpoints, are rejected. `--max-steps` is optional only for OpenRouter, and `--max-turns` is optional only for Claude. Both default to `8` and accept `1` to `32`. `--provider` is -required only for OMP. Other runners reject those OpenRouter, Claude, and OMP -flags. OMP rejects `--max-steps` and `--max-turns`. ACP does not expose a +required only for OMP. `--omp-auth` and `--omp-agent-dir` are also OMP-only. +The directory flag is required for native mode and rejected in API-key mode. +Other runners reject those OpenRouter, Claude, and OMP flags. OMP rejects +`--max-steps` and `--max-turns`. ACP does not expose a portable native model-step boundary, so OMP does not claim an equal step budget with OpenRouter or Claude. Secrets have no command-line flags. A missing required flag or backend credential fails closed. The CLI does not @@ -255,25 +284,33 @@ platforms fail closed. OMP requires the tested `18.1.17` executable. The CLI verifies its SHA-256 and snapshots it once, then starts a fresh local `omp acp` session per trial under -`createLocalHarnessSandbox`. Bootstrap links that snapshot and writes a -session-only `config.yml` and `models.yml` into a disposable home. There is no -Docker engine, container image, or host-socket requirement. - -Each trial writes a static `models.yml` with one private `omp-eval` provider -and one selected-model entry. Public `--provider` / `--model` identity stays -on the report, attempt, and observation. Native ACP uses `--provider omp-eval` -and the original backend model id, including OpenRouter nested slugs. The +`createLocalHarnessSandbox`. Bootstrap links that snapshot. Both authentication +modes load the isolated evaluation settings through an explicit `--config` +overlay. There is no Docker engine, container image, or host-socket requirement. + +Each API-key trial writes a static `models.yml` in its disposable home with one +private `omp-eval` provider and one selected-model entry. Public `--provider` / +`--model` identity stays on the report, attempt, and observation. In API-key mode, +the ACP child is launched with `--provider omp-eval` and the original backend +model id, including OpenRouter nested slugs. The child receives the explicit model key as `OMP_EVAL_PROVIDER_API_KEY` through the stock adapter's environment option. Standard provider credential variables are not inherited. There is no credential broker or request transformation. -OpenAI and OpenRouter use Chat Completions, not the Responses transport that -OMP can select for built-in providers. Anthropic uses its Messages API. The -CLI's `--reasoning` value selects native `--thinking`, with explicit effort +In API-key mode, OpenAI and OpenRouter use Chat Completions, not the Responses +transport that OMP can select for built-in providers. Anthropic uses its +Messages API. The CLI's `--reasoning` value selects native `--thinking`, with explicit effort settings for OpenAI/OpenRouter and thinking budgets for Anthropic. The model entry leaves capacities, cost, and input modalities to OMP 18.1.17's bundled same-id metadata or defaults for unknown model ids. +Native mode passes the requested provider unchanged and uses the selected +profile's models and credentials. It writes no replacement `models.yml` and +passes no evaluator model key. Its agent-directory check reads filesystem +metadata only and rejects missing directories before MCP or model dispatch. +Canonical skills are staged under the disposable `HOME/.agents/skills` in both +modes, so selecting another agent directory does not redirect skill discovery. + The Gina bearer stays on the host. Canonical MCP reads execute in the evaluator process and are exposed as wrapped HarnessAgent host tools. The child does not receive that token. Native OMP builtins still run with ordinary host access; @@ -303,10 +340,20 @@ sessions. The stock Codex adapter also passed a cold local host-tool smoke. Native provider HTTP 500 errors arrive as ACP text plus `end_turn` with no typed metadata, so OMP cannot yet replace every evaluator for reliable provider-failure -classification. Native OAuth subscriptions remain unproven. These synthetic -model/MCP fixtures exercise the real native processes and protocols; they do -not prove real model behavior, production Gina connectivity, or measured -native-plugin activation. +classification. This is tracked in +[OMP #11644](https://github.com/can1357/oh-my-pi/issues/11644). + +Native mode passed a cold synthetic smoke with an inert API key stored through +OMP's stock credential API in a disposable `agent.db`, with no evaluator key +environment variable. The proof covered canonical skill reads, successful and +failed MCP calls, rejection before dispatch for a missing profile, API-key mode +compatibility, and one physical destroy per session. The receipt's +`profile.before` and `profile.after` hashes matched for `models.yml` and +`config.yml`, but `agent.db` changed during native execution. The selected +profile remained in place. This is not a read-only profile mount and proves +stored API-key delegation only, not real OAuth refresh or subscription access. +These fixtures do not prove real model behavior, production Gina connectivity, +or measured native-plugin activation. Each live run writes a v1 aggregate below the ignored `.plugin-eval-runs/` directory and a sibling `.journal-v1.jsonl`, both mode `0600`. OpenRouter diff --git a/packages/evals/__tests__/live-cli.test.ts b/packages/evals/__tests__/live-cli.test.ts index 6f2d7b9..0224fda 100644 --- a/packages/evals/__tests__/live-cli.test.ts +++ b/packages/evals/__tests__/live-cli.test.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; + import * as BunPath from "@effect/platform-bun/BunPath"; import * as BunServices from "@effect/platform-bun/BunServices"; import { assert, describe, it } from "@effect/vitest"; @@ -172,10 +174,11 @@ describe("live eval CLI parser", () => { const omp = yield* parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "anthropic"])); assert.strictEqual(omp.mode, "run"); if (omp.mode === "run" && omp.options.runner === "omp") { - assert.strictEqual(omp.options.provider, "anthropic"); + assert.deepStrictEqual(omp.options.auth, { mode: "api-key", provider: "anthropic" }); assert.strictEqual(omp.options.model, "test-model"); assert.notProperty(omp.options, "maxSteps"); assert.notProperty(omp.options, "maxTurns"); + assert.notProperty(omp.options, "provider"); } const missingProvider = yield* Effect.result(parseLiveEvalCliOptions(requiredFlags("omp"))); @@ -187,6 +190,105 @@ describe("live eval CLI parser", () => { }), ); + it.effect("accepts native OMP auth and rejects mode, path, and runner conflicts", () => + Effect.gen(function* () { + const native = yield* parseLiveEvalCliOptions( + requiredFlags("omp", [ + "--omp-auth", + "native", + "--provider", + "openai-codex", + "--omp-agent-dir", + "/tmp/omp-agent", + ]), + ); + assert.strictEqual(native.mode, "run"); + if (native.mode === "run" && native.options.runner === "omp") { + assert.deepStrictEqual(native.options.auth, { + mode: "native", + provider: "openai-codex", + agentDirectory: "/tmp/omp-agent", + }); + assert.notProperty(native.options, "provider"); + } + + const nativeGoogle = yield* parseLiveEvalCliOptions( + requiredFlags("omp", [ + "--omp-auth", + "native", + "--provider", + "google", + "--omp-agent-dir", + "/tmp/omp-agent", + ]), + ); + assert.strictEqual(nativeGoogle.mode, "run"); + if (nativeGoogle.mode === "run" && nativeGoogle.options.runner === "omp") { + assert.strictEqual(nativeGoogle.options.auth.mode, "native"); + assert.strictEqual(nativeGoogle.options.auth.provider, "google"); + } + + const apiKeyGoogle = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "google"])), + ); + assert.strictEqual(apiKeyGoogle._tag, "Failure"); + + const missingAgentDir = yield* Effect.result( + parseLiveEvalCliOptions( + requiredFlags("omp", ["--omp-auth", "native", "--provider", "openai-codex"]), + ), + ); + assert.strictEqual(missingAgentDir._tag, "Failure"); + + const agentDirOnApiKey = yield* Effect.result( + parseLiveEvalCliOptions( + requiredFlags("omp", ["--provider", "anthropic", "--omp-agent-dir", "/tmp/omp-agent"]), + ), + ); + assert.strictEqual(agentDirOnApiKey._tag, "Failure"); + + const authOnResponses = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("responses", ["--omp-auth", "native"])), + ); + assert.strictEqual(authOnResponses._tag, "Failure"); + + const agentDirOnCodex = yield* Effect.result( + parseLiveEvalCliOptions(requiredFlags("codex", ["--omp-agent-dir", "/tmp/omp-agent"])), + ); + assert.strictEqual(agentDirOnCodex._tag, "Failure"); + + const nativeEndpoint = yield* Effect.result( + parseLiveEvalCliOptions( + requiredFlags("omp", [ + "--omp-auth", + "native", + "--provider", + "openai-codex", + "--omp-agent-dir", + "/tmp/omp-agent", + "--server-url", + PRODUCTION_MCP_URL, + ]), + ), + ); + assert.strictEqual(nativeEndpoint._tag, "Failure"); + + const invalidNativeId = yield* Effect.result( + parseLiveEvalCliOptions( + requiredFlags("omp", [ + "--omp-auth", + "native", + "--provider", + "bad/id", + "--omp-agent-dir", + "/tmp/omp-agent", + ]), + ), + ); + assert.strictEqual(invalidNativeId._tag, "Failure"); + }), + ); + it.effect("requires explicit OpenRouter spend and provider controls", () => Effect.gen(function* () { const flags = requiredFlags("openrouter"); @@ -586,7 +688,10 @@ describe("live eval CLI credentials", () => { it.effect("loads OMP pins without OpenAI keys and hides executable paths", () => Effect.gen(function* () { - const missingKey = yield* loadLiveEvalCredentials("omp").pipe( + const missingKey = yield* loadLiveEvalCredentials({ + runner: "omp", + authMode: "api-key", + }).pipe( withEnv({ ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", OPENAI_API_KEY: "must-not-be-required", @@ -599,7 +704,10 @@ describe("live eval CLI credentials", () => { assert.notInclude(formatLiveEvalCliFailure(missingKey.failure), "must-not-be-required"); } - const relative = yield* loadLiveEvalCredentials("omp").pipe( + const relative = yield* loadLiveEvalCredentials({ + runner: "omp", + authMode: "api-key", + }).pipe( withEnv({ ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", OMP_EVAL_API_KEY: "synthetic-omp-key", @@ -614,7 +722,7 @@ describe("live eval CLI credentials", () => { assert.notInclude(formatLiveEvalCliFailure(relative.failure), "relative/omp"); } - const loaded = yield* loadLiveEvalCredentials("omp").pipe( + const loaded = yield* loadLiveEvalCredentials({ runner: "omp", authMode: "api-key" }).pipe( withEnv({ ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", OMP_EVAL_API_KEY: "synthetic-omp-key", @@ -627,6 +735,8 @@ describe("live eval CLI credentials", () => { if (loaded.runner === "omp") { assert.strictEqual(loaded.executablePath, "/usr/bin/omp"); assert.strictEqual(loaded.expectedSha256, "abcdef"); + assert.strictEqual(loaded.auth.mode, "api-key"); + assert.notProperty(loaded, "apiKey"); } const responses = yield* loadLiveEvalCredentials("responses").pipe( @@ -639,6 +749,61 @@ describe("live eval CLI credentials", () => { assert.strictEqual(responses.runner, "responses"); }), ); + + it.effect("loads native OMP credentials without an API key and still requires Gina", () => + Effect.gen(function* () { + const missingGina = yield* loadLiveEvalCredentials({ + runner: "omp", + authMode: "native", + }).pipe( + withEnv({ + OMP_EVAL_EXECUTABLE: "/usr/bin/omp", + OMP_EVAL_EXECUTABLE_SHA256: "abcdef", + }), + Effect.result, + ); + assert.strictEqual(missingGina._tag, "Failure"); + if (missingGina._tag === "Failure") { + assert.strictEqual(missingGina.failure.reason, "invalid-credentials"); + assert.deepStrictEqual(missingGina.failure.missing, ["ASK_GINA_ACCESS_TOKEN"]); + } + + const loaded = yield* loadLiveEvalCredentials({ + runner: "omp", + authMode: "native", + }).pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OMP_EVAL_EXECUTABLE: "/usr/bin/omp", + OMP_EVAL_EXECUTABLE_SHA256: "AbCDEF", + OPENAI_API_KEY: "must-not-be-required", + }), + ); + assert.strictEqual(loaded.runner, "omp"); + if (loaded.runner === "omp") { + assert.strictEqual(loaded.executablePath, "/usr/bin/omp"); + assert.strictEqual(loaded.expectedSha256, "abcdef"); + assert.deepStrictEqual(loaded.auth, { mode: "native" }); + assert.notProperty(loaded, "apiKey"); + } + + const ambientKey = yield* loadLiveEvalCredentials({ + runner: "omp", + authMode: "native", + }).pipe( + withEnv({ + ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token", + OMP_EVAL_API_KEY: "must-not-be-required", + OMP_EVAL_EXECUTABLE: "/usr/bin/omp", + OMP_EVAL_EXECUTABLE_SHA256: "abcdef", + }), + ); + assert.strictEqual(ambientKey.runner, "omp"); + if (ambientKey.runner === "omp") { + assert.deepStrictEqual(ambientKey.auth, { mode: "native" }); + } + }), + ); }); }); @@ -682,6 +847,133 @@ describe("live eval CLI subprocess", () => { ), ); + it.effect("rejects a missing native profile before authenticated MCP dispatch", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const pathValue = yield* Config.string("PATH"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "live-cli-native-profile-" }); + const cwd = path.join(root, "source"); + const profile = path.join(root, "profile"); + const preload = path.join(root, "block-fetch.mjs"); + const callsPath = path.join(root, "fetch-calls"); + const liveCli = path.join(process.cwd(), "packages/evals/src/bin/live.ts"); + const executablePath = yield* fs.realPath("/bin/echo"); + const executableBytes = yield* fs.readFile(executablePath); + const environment = { PATH: pathValue, HOME: root, GIT_CONFIG_NOSYSTEM: "1" }; + yield* fs.makeDirectory(cwd); + yield* fs.makeDirectory(profile); + yield* fs.makeDirectory(path.join(cwd, "plugins/ask-gina"), { recursive: true }); + yield* fs.copy( + path.join(process.cwd(), "plugins/ask-gina/skills"), + path.join(cwd, "plugins/ask-gina/skills"), + ); + yield* fs.copyFile( + path.join(process.cwd(), "packages/evals/src/fixtures/ask-gina-routing-smoke.yaml"), + path.join(cwd, "suite.yaml"), + ); + yield* fs.writeFileString(path.join(cwd, ".gitignore"), ".plugin-eval-runs/\n"); + yield* fs.writeFileString( + preload, + `import { appendFileSync } from "node:fs"; +const callsPath = new URL("./fetch-calls", import.meta.url); +globalThis.fetch = async () => { + appendFileSync(callsPath, "x"); + throw new Error("native profile fixture blocked fetch"); +}; +`, + ); + // The real entrypoint requires a clean source tree before preparing the OMP runtime. + for (const args of [ + ["init", "--quiet"], + ["add", "."], + [ + "-c", + "user.name=Native profile fixture", + "-c", + "user.email=native-profile@example.invalid", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "--no-gpg-sign", + "-m", + "fixture", + ], + ]) { + const git = yield* ChildProcess.make("git", args, { + cwd, + env: environment, + extendEnv: false, + stdin: "ignore", + stdout: "ignore", + stderr: "pipe", + }); + const [stderr, exitCode] = yield* Effect.all( + [collectBoundedUtf8Output(git.stderr, 65_536), git.exitCode], + { concurrency: "unbounded" }, + ); + assert.strictEqual(exitCode, 0, stderr.text); + } + // Run the control first so an unrelated preflight failure cannot fake zero requests. + for (const [runId, agentDirectory, reachesMcp] of [ + ["run-control", profile, true], + ["run-missing", path.join(root, "missing-profile"), false], + ] as const) { + yield* fs.writeFileString(callsPath, ""); + const child = yield* ChildProcess.make( + "bun", + [ + "--preload", + preload, + liveCli, + ...requiredFlags("omp", [ + "--omp-auth", + "native", + "--provider", + "openai-codex", + "--omp-agent-dir", + agentDirectory, + ]).map((value) => (value === "run-1" ? runId : value)), + ], + { + cwd, + env: { + ...environment, + ASK_GINA_ACCESS_TOKEN: "synthetic-native-profile-token", + OMP_EVAL_EXECUTABLE: executablePath, + OMP_EVAL_EXECUTABLE_SHA256: createHash("sha256") + .update(executableBytes) + .digest("hex"), + }, + extendEnv: false, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }, + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + collectBoundedUtf8Output(child.stdout, 65_536), + collectBoundedUtf8Output(child.stderr, 65_536), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + const output = `${stdout.text}\n${stderr.text}`; + const fetchCalls = (yield* fs.readFileString(callsPath)).length; + assert.notStrictEqual(exitCode, 0, output); + if (reachesMcp) { + assert.isAbove(fetchCalls, 0, output); + } else { + assert.strictEqual(fetchCalls, 0, output); + } + } + }), + ), + ); + it.effect("rejects an unsupported OpenRouter server URL before credentials", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/evals/src/bin/live.ts b/packages/evals/src/bin/live.ts index f67e58c..61e07d9 100755 --- a/packages/evals/src/bin/live.ts +++ b/packages/evals/src/bin/live.ts @@ -56,10 +56,11 @@ import { selectCases, } from "../live"; import { - isOmpProvider, + isOmpApiKeyProvider, + isOmpProviderIdentifier, prepareOmpHarnessRuntime, runOmpHarnessPluginEvalTrial, - type OmpProvider, + type OmpApiKeyProvider, type PluginEvalOmpHarnessError, } from "../omp-harness"; import { @@ -142,7 +143,7 @@ export const liveEvalTrialDispatch = (options: LiveEvalCliOptions): LiveEvalTria return { target, maxTurns: options.maxTurns }; } if (options.runner === "omp") { - return { target, model: `${options.provider}/${options.model}` }; + return { target, model: `${options.auth.provider}/${options.model}` }; } return { target, displayedModel: options.model }; }; @@ -185,7 +186,12 @@ export type LiveEvalCliOptions = readonly expectedProvider: string; }) | (LiveEvalCliSharedOptions & { readonly runner: "claude"; readonly maxTurns: number }) - | (LiveEvalCliSharedOptions & { readonly runner: "omp"; readonly provider: OmpProvider }); + | (LiveEvalCliSharedOptions & { + readonly runner: "omp"; + readonly auth: + | { readonly mode: "api-key"; readonly provider: OmpApiKeyProvider } + | { readonly mode: "native"; readonly provider: string; readonly agentDirectory: string }; + }); export type LiveEvalCliParseResult = | { readonly mode: "help"; readonly usage: string } @@ -239,14 +245,14 @@ export const formatLiveEvalCliUsage = (runner?: LiveEvalRunner): string => { } if (runner === "omp") { return [ - `Usage: bun run eval:omp -- ${REQUIRED_LIVE_EVAL_FLAGS} --provider `, - `Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OMP_EVAL_API_KEY}, ${OMP_EVAL_EXECUTABLE}, ${OMP_EVAL_EXECUTABLE_SHA256}`, + `Usage: bun run eval:omp -- ${REQUIRED_LIVE_EVAL_FLAGS} --provider [--omp-auth api-key|native] [--omp-agent-dir ]`, + `--omp-auth default api-key. --provider for api-key: openai|anthropic|openrouter. --omp-agent-dir is required for native and forbidden for api-key. Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OMP_EVAL_EXECUTABLE}, ${OMP_EVAL_EXECUTABLE_SHA256}; ${OMP_EVAL_API_KEY} (api-key).`, ].join("\n"); } return [ `Usage: bun run eval: -- ${REQUIRED_LIVE_EVAL_FLAGS}`, - `OpenRouter-only: --openrouter-endpoint --expected-provider