From cf69eb8ba0f896e3ae68086deeea3087b4afab6a Mon Sep 17 00:00:00 2001
From: Eric Juta <ericjohnjuta@gmail.com>
Date: Fri, 11 Sep 2026 02:37:44 +0000
Subject: [PATCH 1/4] feat(evals): support native OMP profile authentication

Add explicit native and API-key auth modes without copying login stores or generating native provider credentials. Keep canonical skills in the disposable session home, reject missing profile directories before dispatch, and retain existing API-key mode.
---
 packages/evals/__tests__/live-cli.test.ts | 171 +++++++++++++-
 packages/evals/src/bin/live.ts            | 130 ++++++++---
 packages/evals/src/index.ts               |   3 +-
 packages/evals/src/omp-harness.ts         | 259 ++++++++++++++--------
 4 files changed, 435 insertions(+), 128 deletions(-)

diff --git a/packages/evals/__tests__/live-cli.test.ts b/packages/evals/__tests__/live-cli.test.ts
index 6f2d7b9..4199d0a 100644
--- a/packages/evals/__tests__/live-cli.test.ts
+++ b/packages/evals/__tests__/live-cli.test.ts
@@ -172,10 +172,11 @@ describe("live eval CLI parser", () => {
       const omp = yield* parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "anthropic"]));
       assert.strictEqual(omp.mode, "run");
       if (omp.mode === "run" && omp.options.runner === "omp") {
-        assert.strictEqual(omp.options.provider, "anthropic");
+        assert.deepStrictEqual(omp.options.auth, { mode: "api-key", provider: "anthropic" });
         assert.strictEqual(omp.options.model, "test-model");
         assert.notProperty(omp.options, "maxSteps");
         assert.notProperty(omp.options, "maxTurns");
+        assert.notProperty(omp.options, "provider");
       }
 
       const missingProvider = yield* Effect.result(parseLiveEvalCliOptions(requiredFlags("omp")));
@@ -187,6 +188,105 @@ describe("live eval CLI parser", () => {
     }),
   );
 
+  it.effect("accepts native OMP auth and rejects mode, path, and runner conflicts", () =>
+    Effect.gen(function* () {
+      const native = yield* parseLiveEvalCliOptions(
+        requiredFlags("omp", [
+          "--omp-auth",
+          "native",
+          "--provider",
+          "openai-codex",
+          "--omp-agent-dir",
+          "/tmp/omp-agent",
+        ]),
+      );
+      assert.strictEqual(native.mode, "run");
+      if (native.mode === "run" && native.options.runner === "omp") {
+        assert.deepStrictEqual(native.options.auth, {
+          mode: "native",
+          provider: "openai-codex",
+          agentDirectory: "/tmp/omp-agent",
+        });
+        assert.notProperty(native.options, "provider");
+      }
+
+      const nativeGoogle = yield* parseLiveEvalCliOptions(
+        requiredFlags("omp", [
+          "--omp-auth",
+          "native",
+          "--provider",
+          "google",
+          "--omp-agent-dir",
+          "/tmp/omp-agent",
+        ]),
+      );
+      assert.strictEqual(nativeGoogle.mode, "run");
+      if (nativeGoogle.mode === "run" && nativeGoogle.options.runner === "omp") {
+        assert.strictEqual(nativeGoogle.options.auth.mode, "native");
+        assert.strictEqual(nativeGoogle.options.auth.provider, "google");
+      }
+
+      const apiKeyGoogle = yield* Effect.result(
+        parseLiveEvalCliOptions(requiredFlags("omp", ["--provider", "google"])),
+      );
+      assert.strictEqual(apiKeyGoogle._tag, "Failure");
+
+      const missingAgentDir = yield* Effect.result(
+        parseLiveEvalCliOptions(
+          requiredFlags("omp", ["--omp-auth", "native", "--provider", "openai-codex"]),
+        ),
+      );
+      assert.strictEqual(missingAgentDir._tag, "Failure");
+
+      const agentDirOnApiKey = yield* Effect.result(
+        parseLiveEvalCliOptions(
+          requiredFlags("omp", ["--provider", "anthropic", "--omp-agent-dir", "/tmp/omp-agent"]),
+        ),
+      );
+      assert.strictEqual(agentDirOnApiKey._tag, "Failure");
+
+      const authOnResponses = yield* Effect.result(
+        parseLiveEvalCliOptions(requiredFlags("responses", ["--omp-auth", "native"])),
+      );
+      assert.strictEqual(authOnResponses._tag, "Failure");
+
+      const agentDirOnCodex = yield* Effect.result(
+        parseLiveEvalCliOptions(requiredFlags("codex", ["--omp-agent-dir", "/tmp/omp-agent"])),
+      );
+      assert.strictEqual(agentDirOnCodex._tag, "Failure");
+
+      const nativeEndpoint = yield* Effect.result(
+        parseLiveEvalCliOptions(
+          requiredFlags("omp", [
+            "--omp-auth",
+            "native",
+            "--provider",
+            "openai-codex",
+            "--omp-agent-dir",
+            "/tmp/omp-agent",
+            "--server-url",
+            PRODUCTION_MCP_URL,
+          ]),
+        ),
+      );
+      assert.strictEqual(nativeEndpoint._tag, "Failure");
+
+      const invalidNativeId = yield* Effect.result(
+        parseLiveEvalCliOptions(
+          requiredFlags("omp", [
+            "--omp-auth",
+            "native",
+            "--provider",
+            "bad/id",
+            "--omp-agent-dir",
+            "/tmp/omp-agent",
+          ]),
+        ),
+      );
+      assert.strictEqual(invalidNativeId._tag, "Failure");
+    }),
+  );
+
   it.effect("requires explicit OpenRouter spend and provider controls", () =>
     Effect.gen(function* () {
       const flags = requiredFlags("openrouter");
@@ -586,7 +686,10 @@ describe("live eval CLI credentials", () => {
 
     it.effect("loads OMP pins without OpenAI keys and hides executable paths", () =>
       Effect.gen(function* () {
-        const missingKey = yield* loadLiveEvalCredentials("omp").pipe(
+        const missingKey = yield* loadLiveEvalCredentials({
+          runner: "omp",
+          authMode: "api-key",
+        }).pipe(
           withEnv({
             ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token",
             OPENAI_API_KEY: "must-not-be-required",
@@ -599,7 +702,10 @@ describe("live eval CLI credentials", () => {
           assert.notInclude(formatLiveEvalCliFailure(missingKey.failure), "must-not-be-required");
         }
 
-        const relative = yield* loadLiveEvalCredentials("omp").pipe(
+        const relative = yield* loadLiveEvalCredentials({
+          runner: "omp",
+          authMode: "api-key",
+        }).pipe(
           withEnv({
             ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token",
             OMP_EVAL_API_KEY: "synthetic-omp-key",
@@ -614,7 +720,7 @@ describe("live eval CLI credentials", () => {
           assert.notInclude(formatLiveEvalCliFailure(relative.failure), "relative/omp");
         }
 
-        const loaded = yield* loadLiveEvalCredentials("omp").pipe(
+        const loaded = yield* loadLiveEvalCredentials({ runner: "omp", authMode: "api-key" }).pipe(
           withEnv({
             ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token",
             OMP_EVAL_API_KEY: "synthetic-omp-key",
@@ -627,6 +733,8 @@ describe("live eval CLI credentials", () => {
         if (loaded.runner === "omp") {
           assert.strictEqual(loaded.executablePath, "/usr/bin/omp");
           assert.strictEqual(loaded.expectedSha256, "abcdef");
+          assert.strictEqual(loaded.auth.mode, "api-key");
+          assert.notProperty(loaded, "apiKey");
         }
 
         const responses = yield* loadLiveEvalCredentials("responses").pipe(
@@ -639,6 +747,61 @@ describe("live eval CLI credentials", () => {
         assert.strictEqual(responses.runner, "responses");
       }),
     );
+
+    it.effect("loads native OMP credentials without an API key and still requires Gina", () =>
+      Effect.gen(function* () {
+        const missingGina = yield* loadLiveEvalCredentials({
+          runner: "omp",
+          authMode: "native",
+        }).pipe(
+          withEnv({
+            OMP_EVAL_EXECUTABLE: "/usr/bin/omp",
+            OMP_EVAL_EXECUTABLE_SHA256: "abcdef",
+          }),
+          Effect.result,
+        );
+        assert.strictEqual(missingGina._tag, "Failure");
+        if (missingGina._tag === "Failure") {
+          assert.strictEqual(missingGina.failure.reason, "invalid-credentials");
+          assert.deepStrictEqual(missingGina.failure.missing, ["ASK_GINA_ACCESS_TOKEN"]);
+        }
+
+        const loaded = yield* loadLiveEvalCredentials({
+          runner: "omp",
+          authMode: "native",
+        }).pipe(
+          withEnv({
+            ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token",
+            OMP_EVAL_EXECUTABLE: "/usr/bin/omp",
+            OMP_EVAL_EXECUTABLE_SHA256: "AbCDEF",
+            OPENAI_API_KEY: "must-not-be-required",
+          }),
+        );
+        assert.strictEqual(loaded.runner, "omp");
+        if (loaded.runner === "omp") {
+          assert.strictEqual(loaded.executablePath, "/usr/bin/omp");
+          assert.strictEqual(loaded.expectedSha256, "abcdef");
+          assert.deepStrictEqual(loaded.auth, { mode: "native" });
+          assert.notProperty(loaded, "apiKey");
+        }
+
+        const ambientKey = yield* loadLiveEvalCredentials({
+          runner: "omp",
+          authMode: "native",
+        }).pipe(
+          withEnv({
+            ASK_GINA_ACCESS_TOKEN: "synthetic-gina-token",
+            OMP_EVAL_API_KEY: "must-not-be-required",
+            OMP_EVAL_EXECUTABLE: "/usr/bin/omp",
+            OMP_EVAL_EXECUTABLE_SHA256: "abcdef",
+          }),
+        );
+        assert.strictEqual(ambientKey.runner, "omp");
+        if (ambientKey.runner === "omp") {
+          assert.deepStrictEqual(ambientKey.auth, { mode: "native" });
+        }
+      }),
+    );
   });
 });
 
diff --git a/packages/evals/src/bin/live.ts b/packages/evals/src/bin/live.ts
index f67e58c..8568650 100755
--- a/packages/evals/src/bin/live.ts
+++ b/packages/evals/src/bin/live.ts
@@ -56,10 +56,11 @@ import {
   selectCases,
 } from "../live";
 import {
-  isOmpProvider,
+  isOmpApiKeyProvider,
+  isOmpProviderIdentifier,
   prepareOmpHarnessRuntime,
   runOmpHarnessPluginEvalTrial,
-  type OmpProvider,
+  type OmpApiKeyProvider,
   type PluginEvalOmpHarnessError,
 } from "../omp-harness";
 import {
@@ -142,7 +143,7 @@ export const liveEvalTrialDispatch = (options: LiveEvalCliOptions): LiveEvalTria
     return { target, maxTurns: options.maxTurns };
   }
   if (options.runner === "omp") {
-    return { target, model: `${options.provider}/${options.model}` };
+    return { target, model: `${options.auth.provider}/${options.model}` };
   }
   return { target, displayedModel: options.model };
 };
@@ -185,7 +186,12 @@ export type LiveEvalCliOptions =
       readonly expectedProvider: string;
     })
   | (LiveEvalCliSharedOptions & { readonly runner: "claude"; readonly maxTurns: number })
-  | (LiveEvalCliSharedOptions & { readonly runner: "omp"; readonly provider: OmpProvider });
+  | (LiveEvalCliSharedOptions & {
+      readonly runner: "omp";
+      readonly auth:
+        | { readonly mode: "api-key"; readonly provider: OmpApiKeyProvider }
+        | { readonly mode: "native"; readonly provider: string; readonly agentDirectory: string };
+    });
 
 export type LiveEvalCliParseResult =
   | { readonly mode: "help"; readonly usage: string }
@@ -239,14 +245,14 @@ export const formatLiveEvalCliUsage = (runner?: LiveEvalRunner): string => {
   }
   if (runner === "omp") {
     return [
-      `Usage: bun run eval:omp -- ${REQUIRED_LIVE_EVAL_FLAGS} --provider <openai|anthropic|openrouter>`,
-      `Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OMP_EVAL_API_KEY}, ${OMP_EVAL_EXECUTABLE}, ${OMP_EVAL_EXECUTABLE_SHA256}`,
+      `Usage: bun run eval:omp -- ${REQUIRED_LIVE_EVAL_FLAGS} --provider <id> [--omp-auth api-key|native] [--omp-agent-dir <dir>]`,
+      `--omp-auth default api-key. --provider for api-key: openai|anthropic|openrouter. --omp-agent-dir is required for native and forbidden for api-key. Environment: ${ASK_GINA_ACCESS_TOKEN}, ${OMP_EVAL_EXECUTABLE}, ${OMP_EVAL_EXECUTABLE_SHA256}; ${OMP_EVAL_API_KEY} (api-key).`,
     ].join("\n");
   }
   return [
     `Usage: bun run eval:<responses|codex|openrouter|claude|omp> -- ${REQUIRED_LIVE_EVAL_FLAGS}`,
-    `OpenRouter-only: --openrouter-endpoint <slug> --expected-provider <label> --max-cost-usd <amount> [--max-steps <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_OPENROUTER_MAX_STEPS}) [--server-url <${PRODUCTION_MCP_URL}|${ALPHA_GINA_READ_SERVER_URL}>] (default ${PRODUCTION_MCP_URL}). Claude-only: [--max-turns <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_CLAUDE_MAX_TURNS}). OMP-only: --provider <openai|anthropic|openrouter>.`,
-    `Environment: ${ASK_GINA_ACCESS_TOKEN} always; ${OPENAI_API_KEY} (responses, codex); ${OPENROUTER_API_KEY} (openrouter); ${ANTHROPIC_API_KEY} and ${CLAUDE_EVAL_EXECUTABLE} (claude); ${CODEX_EVAL_EXECUTABLE} and ${CODEX_EVAL_EXECUTABLE_SHA256} (codex); ${OMP_EVAL_API_KEY}, ${OMP_EVAL_EXECUTABLE}, and ${OMP_EVAL_EXECUTABLE_SHA256} (omp).`,
+    `OpenRouter-only: --openrouter-endpoint <slug> --expected-provider <label> --max-cost-usd <amount> [--max-steps <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_OPENROUTER_MAX_STEPS}) [--server-url <${PRODUCTION_MCP_URL}|${ALPHA_GINA_READ_SERVER_URL}>] (default ${PRODUCTION_MCP_URL}). Claude-only: [--max-turns <1..${MAXIMUM_LIVE_EVAL_TOOL_BUDGET}>] (default ${DEFAULT_CLAUDE_MAX_TURNS}). OMP-only: --provider <id> [--omp-auth api-key|native] [--omp-agent-dir <dir>].`,
+    `Environment: ${ASK_GINA_ACCESS_TOKEN} always; ${OPENAI_API_KEY} (responses, codex); ${OPENROUTER_API_KEY} (openrouter); ${ANTHROPIC_API_KEY} and ${CLAUDE_EVAL_EXECUTABLE} (claude); ${CODEX_EVAL_EXECUTABLE} and ${CODEX_EVAL_EXECUTABLE_SHA256} (codex); ${OMP_EVAL_EXECUTABLE} and ${OMP_EVAL_EXECUTABLE_SHA256} (omp); ${OMP_EVAL_API_KEY} (omp api-key).`,
   ].join("\n");
 };
 
@@ -286,7 +292,9 @@ export const parseLiveEvalCliOptions = (
     let attemptsOutputPath: string | undefined;
     let maxSteps: number | undefined;
     let maxTurns: number | undefined;
-    let provider: OmpProvider | undefined;
+    let provider: string | undefined;
+    let ompAuth: "api-key" | "native" | undefined;
+    let ompAgentDir: string | undefined;
     let endpoint: string | undefined;
     let serverUrl: string | undefined;
     let maxCostUsd: number | undefined;
@@ -365,11 +373,17 @@ export const parseLiveEvalCliOptions = (
           }
           break;
         case "--provider":
-          if (isOmpProvider(value)) {
-            provider = value;
-            break;
+          provider = value;
+          break;
+        case "--omp-auth":
+          if (value !== "api-key" && value !== "native") {
+            return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
           }
-          return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
+          ompAuth = value;
+          break;
+        case "--omp-agent-dir":
+          ompAgentDir = value;
+          break;
         case "--openrouter-endpoint":
           endpoint = value;
           break;
@@ -423,7 +437,10 @@ export const parseLiveEvalCliOptions = (
     if (runner !== "claude" && seenFlags.has("--max-turns")) {
       return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
     }
-    if (runner !== "omp" && seenFlags.has("--provider")) {
+    if (
+      runner !== "omp" &&
+      ["--provider", "--omp-auth", "--omp-agent-dir"].some((flag) => seenFlags.has(flag))
+    ) {
       return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
     }
     if (runner !== "openrouter" && seenFlags.has("--openrouter-endpoint")) {
@@ -482,12 +499,29 @@ export const parseLiveEvalCliOptions = (
       };
     }
     if (runner === "omp") {
+      const authMode = ompAuth ?? "api-key";
       if (provider === undefined) {
         return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
       }
+      if (authMode === "native") {
+        if (ompAgentDir === undefined || !isOmpProviderIdentifier(provider)) {
+          return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
+        }
+        return {
+          mode: "run",
+          options: {
+            ...shared,
+            runner,
+            auth: { mode: "native", provider, agentDirectory: ompAgentDir },
+          },
+        };
+      }
+      if (ompAgentDir !== undefined || !isOmpApiKeyProvider(provider)) {
+        return yield* new LiveEvalCliError({ reason: "invalid-arguments" });
+      }
       return {
         mode: "run",
-        options: { ...shared, runner, provider },
+        options: { ...shared, runner, auth: { mode: "api-key", provider } },
       };
     }
     return { mode: "run", options: { ...shared, runner } };
@@ -597,16 +631,41 @@ export type LiveEvalCredentials =
   | {
       readonly runner: "omp";
       readonly accessToken: Redacted.Redacted<string>;
-      readonly apiKey: Redacted.Redacted<string>;
       readonly executablePath: string;
       readonly expectedSha256: string;
+      readonly auth:
+        | { readonly mode: "api-key"; readonly apiKey: Redacted.Redacted<string> }
+        | { readonly mode: "native" };
     };
 
+export type LiveEvalCredentialRequest =
+  | Exclude<LiveEvalRunner, "omp">
+  | { readonly runner: "omp"; readonly authMode: "api-key" | "native" };
+
 export const loadLiveEvalCredentials = (
-  runner: LiveEvalRunner,
+  request: LiveEvalCredentialRequest,
 ): Effect.Effect<LiveEvalCredentials, LiveEvalCliError, Path.Path> =>
   Effect.gen(function* () {
     const accessToken = yield* loadRedactedEnv(ASK_GINA_ACCESS_TOKEN);
+    if (typeof request !== "string") {
+      const path = yield* Path.Path;
+      const apiKey =
+        request.authMode === "native" ? undefined : yield* loadRedactedEnv(OMP_EVAL_API_KEY);
+      const executablePath = yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE);
+      const expectedSha256 = (yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE_SHA256)).toLowerCase();
+      if (!path.isAbsolute(executablePath)) {
+        return yield* missingCredentials([OMP_EVAL_EXECUTABLE]);
+      }
+      return {
+        runner: "omp",
+        accessToken,
+        executablePath,
+        expectedSha256,
+        auth:
+          apiKey === undefined ? { mode: "native" as const } : { mode: "api-key" as const, apiKey },
+      };
+    }
+    const runner = request;
     if (runner === "responses") {
       const openAiApiKey = yield* loadRedactedEnv(OPENAI_API_KEY);
       return { runner, accessToken, openAiApiKey };
@@ -624,16 +683,6 @@ export const loadLiveEvalCredentials = (
       }
       return { runner, accessToken, apiKey, executablePath };
     }
-    if (runner === "omp") {
-      const path = yield* Path.Path;
-      const apiKey = yield* loadRedactedEnv(OMP_EVAL_API_KEY);
-      const executablePath = yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE);
-      const expectedSha256 = (yield* loadNonEmptyEnv(OMP_EVAL_EXECUTABLE_SHA256)).toLowerCase();
-      if (!path.isAbsolute(executablePath)) {
-        return yield* missingCredentials([OMP_EVAL_EXECUTABLE]);
-      }
-      return { runner, accessToken, apiKey, executablePath, expectedSha256 };
-    }
     const openAiApiKey = yield* loadRedactedEnv(OPENAI_API_KEY);
     const executablePath = yield* loadNonEmptyEnv(CODEX_EVAL_EXECUTABLE);
     const expectedSha256 = (yield* loadNonEmptyEnv(CODEX_EVAL_EXECUTABLE_SHA256)).toLowerCase();
@@ -1242,7 +1291,9 @@ const run = (options: LiveEvalCliOptions) =>
         caseIds: plannedCaseIds,
         repetitions: options.repetitions,
       });
-      const credentials = yield* loadLiveEvalCredentials(options.runner);
+      const credentials = yield* loadLiveEvalCredentials(
+        options.runner === "omp" ? { runner: "omp", authMode: options.auth.mode } : options.runner,
+      );
       const isolatedEnvironment = yield* loadCodexEnvironment();
       yield* requireCleanSource(root, isolatedEnvironment);
       let codexRuntime: CodexEvalRuntime | undefined;
@@ -1421,19 +1472,35 @@ const run = (options: LiveEvalCliOptions) =>
                       availableTools,
                       timeoutMs: options.timeoutMs,
                     });
-                  case "omp":
+                  case "omp": {
                     if (credentials.runner !== "omp" || ompRuntimeDirectory === undefined) {
                       return Effect.fail(new LiveEvalCliError({ reason: "omp-preflight-failed" }));
                     }
+                    const auth =
+                      options.auth.mode === "native"
+                        ? {
+                            mode: "native" as const,
+                            provider: options.auth.provider,
+                            agentDirectory: options.auth.agentDirectory,
+                          }
+                        : credentials.auth.mode === "api-key"
+                          ? {
+                              mode: "api-key" as const,
+                              provider: options.auth.provider,
+                              apiKey: credentials.auth.apiKey,
+                            }
+                          : undefined;
+                    if (auth === undefined) {
+                      return Effect.fail(new LiveEvalCliError({ reason: "omp-preflight-failed" }));
+                    }
                     return runOmpHarnessPluginEvalTrial(input.evalCase, {
                       runId: input.runId,
                       repetition: input.repetition,
                       availableTools,
                       runtimeDirectory: ompRuntimeDirectory,
-                      provider: options.provider,
+                      auth,
                       model: options.model,
                       reasoning: options.reasoning,
-                      apiKey: credentials.apiKey,
                       mcpAuthorization: credentials.accessToken,
                       timeoutMs: options.timeoutMs,
                     }).pipe(
@@ -1442,6 +1509,7 @@ const run = (options: LiveEvalCliOptions) =>
                         () => new LiveEvalCliError({ reason: "trial-failed" }),
                       ),
                     );
+                  }
                 }
               },
             ).pipe(
diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts
index 68f8186..a87d48a 100644
--- a/packages/evals/src/index.ts
+++ b/packages/evals/src/index.ts
@@ -132,7 +132,8 @@ export {
   type PrepareOmpHarnessRuntimeOptions,
   type PreparedOmpHarnessRuntime,
   type OmpHarnessTrialOptions,
-  type OmpProvider,
+  type OmpApiKeyProvider,
+  type OmpAuth,
   type PluginEvalOmpHarnessError,
 } from "./omp-harness";
 export {
diff --git a/packages/evals/src/omp-harness.ts b/packages/evals/src/omp-harness.ts
index 9db0a19..78b43c2 100644
--- a/packages/evals/src/omp-harness.ts
+++ b/packages/evals/src/omp-harness.ts
@@ -78,12 +78,17 @@ const CANONICAL_TOOL_NAMES: Record<string, true> = Object.fromEntries(
   CANONICAL_ALLOWED_TOOLS.map((name) => [name, true as const]),
 );
 
-export type OmpProvider = keyof typeof PROVIDER_PROFILE;
+export type OmpApiKeyProvider = keyof typeof PROVIDER_PROFILE;
 export type OmpReasoning = keyof typeof OMP_REASONING;
 
-export const isOmpProvider = (value: string): value is OmpProvider =>
+export const isOmpApiKeyProvider = (value: string): value is OmpApiKeyProvider =>
   Object.hasOwn(PROVIDER_PROFILE, value);
 
+const OMP_PROVIDER_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
+
+export const isOmpProviderIdentifier = (value: string): boolean =>
+  OMP_PROVIDER_IDENTIFIER.test(value);
+
 const isOmpReasoning = (value: string): value is OmpReasoning =>
   Object.hasOwn(OMP_REASONING, value);
 
@@ -97,36 +102,58 @@ export interface PreparedOmpHarnessRuntime {
   readonly runtimeDirectory: string;
 }
 
+export type OmpAuth =
+  | {
+      readonly mode: "api-key";
+      readonly provider: OmpApiKeyProvider;
+      readonly apiKey: Redacted.Redacted<string>;
+      readonly providerBaseUrl?: string;
+    }
+  | {
+      readonly mode: "native";
+      readonly provider: string;
+      readonly agentDirectory: string;
+    };
+
 export interface OmpHarnessTrialOptions {
   readonly runId: string;
   readonly repetition: number;
   readonly availableTools: readonly string[];
   readonly runtimeDirectory: string;
-  readonly provider: OmpProvider;
+  readonly auth: OmpAuth;
   readonly model: string;
   readonly reasoning: string;
-  readonly apiKey: Redacted.Redacted<string>;
   readonly mcpAuthorization: Redacted.Redacted<string>;
   readonly timeoutMs: number;
   readonly serverUrl?: string;
-  readonly providerBaseUrl?: string;
   readonly sandbox?: HarnessV1SandboxProvider;
 }
 
+type ValidatedOmpAuth =
+  | {
+      readonly mode: "api-key";
+      readonly provider: OmpApiKeyProvider;
+      readonly apiKey: string;
+      readonly providerBaseUrl?: string;
+    }
+  | {
+      readonly mode: "native";
+      readonly provider: string;
+      readonly agentDirectory: string;
+    };
+
 interface ValidatedOmpHarnessTrialOptions {
   readonly runId: string;
   readonly repetition: number;
   readonly availableTools: readonly string[];
   readonly runtimeDirectory: string;
-  readonly provider: OmpProvider;
+  readonly auth: ValidatedOmpAuth;
   readonly model: string;
   readonly modelIdentity: string;
   readonly reasoning: OmpReasoning;
-  readonly apiKey: string;
   readonly mcpAuthorization: string;
   readonly timeoutMs: number;
   readonly serverUrl: string;
-  readonly providerBaseUrl?: string;
   readonly sandbox?: HarnessV1SandboxProvider;
 }
 
@@ -509,9 +536,9 @@ const nestedEvalConfig = [
   "  enableCodexUser: false",
   "  enableClaudeUser: false",
   "  enableClaudeProject: false",
-  "  enablePiUser: true",
+  "  enablePiUser: false",
   "  enablePiProject: false",
-  "  enableAgentsUser: false",
+  "  enableAgentsUser: true",
   "  enableAgentsProject: false",
   "  includeSkills:",
   ...SKILL_NAMES.map((name) => `    - ${name}`),
@@ -519,7 +546,7 @@ const nestedEvalConfig = [
 ].join("\n");
 
 const modelsYaml = (
-  provider: OmpProvider,
+  provider: OmpApiKeyProvider,
   model: string,
   providerBaseUrl: string | undefined,
 ): string => {
@@ -553,22 +580,27 @@ const modelsYaml = (
 };
 
 const installCommand = (
-  provider: OmpProvider,
-  model: string,
-  providerBaseUrl: string | undefined,
   runtimeDirectory: string,
+  auth: ValidatedOmpAuth,
+  model: string,
 ): string => {
-  const models = modelsYaml(provider, model, providerBaseUrl);
-  return [
-    'mkdir -p "$HOME/.local/bin" "$HOME/.omp/agent"',
+  const lines = [
+    'mkdir -p "$HOME/.local/bin"',
     `ln -sfn ${shellQuote(`${runtimeDirectory}/omp`)} "$HOME/.local/bin/omp"`,
-    `cp ${shellQuote(`${runtimeDirectory}/config.yml`)} "$HOME/.omp/agent/config.yml"`,
-    "cat > \"$HOME/.omp/agent/models.yml\" <<'OMP_EVAL_MODELS_YML'",
-    models.trimEnd(),
-    "OMP_EVAL_MODELS_YML",
+  ];
+  if (auth.mode === "api-key") {
+    lines.push(
+      'mkdir -p "$HOME/.omp/agent"',
+      "cat > \"$HOME/.omp/agent/models.yml\" <<'OMP_EVAL_MODELS_YML'",
+      modelsYaml(auth.provider, model, auth.providerBaseUrl).trimEnd(),
+      "OMP_EVAL_MODELS_YML",
+    );
+  }
+  lines.push(
     'version="$("$HOME/.local/bin/omp" --version)"',
     'case "$version" in omp/*) ;; *) exit 1 ;; esac',
-  ].join("\n");
+  );
+  return lines.join("\n");
 };
 
 const snapshotExecutable = (
@@ -832,69 +864,108 @@ export const prepareOmpHarnessRuntime = (
     return { runtimeDirectory };
   });
 
+const invalidOptions = (caseId: string): PluginEvalOmpHarnessRequestError =>
+  new PluginEvalOmpHarnessRequestError({ caseId, reason: "invalid-options" });
+
 const validateOptions = (
   evalCase: PluginEvalCase,
   options: OmpHarnessTrialOptions,
-): Effect.Effect<ValidatedOmpHarnessTrialOptions, PluginEvalOmpHarnessRequestError> => {
-  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const apiKey = Redacted.value(options.apiKey);
-  const mcpAuthorization = Redacted.value(options.mcpAuthorization);
-  const model = options.model.trim();
-  const runId = options.runId.trim();
-  const providerBaseUrl =
-    options.providerBaseUrl === undefined
-      ? undefined
-      : parseProviderBaseUrl(options.providerBaseUrl);
-  const modelIdentity = `${options.provider}/${model}`;
-  const serverUrl = options.serverUrl ?? PRODUCTION_MCP_URL;
-  if (options.providerBaseUrl !== undefined && providerBaseUrl === undefined) {
-    return Effect.fail(
-      new PluginEvalOmpHarnessRequestError({ caseId: evalCase.id, reason: "invalid-endpoint" }),
-    );
-  }
-  if (
-    !isOmpProvider(options.provider) ||
-    !catalogsMatch(options.availableTools, CANONICAL_ALLOWED_TOOLS) ||
-    runId.length === 0 ||
-    model.length === 0 ||
-    apiKey.trim().length === 0 ||
-    mcpAuthorization.trim().length === 0 ||
-    !options.runtimeDirectory.startsWith("/") ||
-    !Number.isSafeInteger(options.repetition) ||
-    options.repetition <= 0 ||
-    !Number.isSafeInteger(timeoutMs) ||
-    timeoutMs <= 0 ||
-    (options.serverUrl !== undefined && options.serverUrl.trim().length === 0)
-  ) {
-    return Effect.fail(
-      new PluginEvalOmpHarnessRequestError({ caseId: evalCase.id, reason: "invalid-options" }),
-    );
-  }
-  if (!isOmpReasoning(options.reasoning)) {
-    return Effect.fail(
-      new PluginEvalOmpHarnessRequestError({
+): Effect.Effect<
+  ValidatedOmpHarnessTrialOptions,
+  PluginEvalOmpHarnessRequestError,
+  FileSystem.FileSystem | Path.Path
+> =>
+  Effect.gen(function* () {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const mcpAuthorization = Redacted.value(options.mcpAuthorization);
+    const model = options.model.trim();
+    const runId = options.runId.trim();
+    const serverUrl = options.serverUrl ?? PRODUCTION_MCP_URL;
+    const authOption = options.auth;
+    if (
+      authOption === undefined ||
+      typeof authOption !== "object" ||
+      authOption === null ||
+      (authOption.mode !== "api-key" && authOption.mode !== "native") ||
+      !catalogsMatch(options.availableTools, CANONICAL_ALLOWED_TOOLS) ||
+      runId.length === 0 ||
+      model.length === 0 ||
+      mcpAuthorization.trim().length === 0 ||
+      !options.runtimeDirectory.startsWith("/") ||
+      !Number.isSafeInteger(options.repetition) ||
+      options.repetition <= 0 ||
+      !Number.isSafeInteger(timeoutMs) ||
+      timeoutMs <= 0 ||
+      (options.serverUrl !== undefined && options.serverUrl.trim().length === 0)
+    ) {
+      return yield* invalidOptions(evalCase.id);
+    }
+    if (!isOmpReasoning(options.reasoning)) {
+      return yield* new PluginEvalOmpHarnessRequestError({
         caseId: evalCase.id,
         reason: "unsupported-reasoning",
-      }),
-    );
-  }
-  return Effect.succeed({
-    runId,
-    repetition: options.repetition,
-    availableTools: [...CANONICAL_ALLOWED_TOOLS],
-    runtimeDirectory: options.runtimeDirectory,
-    provider: options.provider,
-    model,
-    modelIdentity,
-    reasoning: options.reasoning,
-    apiKey,
-    mcpAuthorization,
-    timeoutMs,
-    serverUrl,
-    ...(providerBaseUrl === undefined ? {} : { providerBaseUrl }),
-    ...(options.sandbox === undefined ? {} : { sandbox: options.sandbox }),
+      });
+    }
+
+    let auth: ValidatedOmpAuth;
+    if (authOption.mode === "api-key") {
+      const providerBaseUrl =
+        authOption.providerBaseUrl === undefined
+          ? undefined
+          : parseProviderBaseUrl(authOption.providerBaseUrl);
+      if (authOption.providerBaseUrl !== undefined && providerBaseUrl === undefined) {
+        return yield* new PluginEvalOmpHarnessRequestError({
+          caseId: evalCase.id,
+          reason: "invalid-endpoint",
+        });
+      }
+      const apiKey = Redacted.value(authOption.apiKey);
+      if (!isOmpApiKeyProvider(authOption.provider) || apiKey.trim().length === 0) {
+        return yield* invalidOptions(evalCase.id);
+      }
+      auth = {
+        mode: "api-key",
+        provider: authOption.provider,
+        apiKey,
+        ...(providerBaseUrl === undefined ? {} : { providerBaseUrl }),
+      };
+    } else {
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      if (
+        !isOmpProviderIdentifier(authOption.provider) ||
+        !path.isAbsolute(authOption.agentDirectory)
+      ) {
+        return yield* invalidOptions(evalCase.id);
+      }
+      const info = yield* fs
+        .stat(authOption.agentDirectory)
+        .pipe(Effect.mapError(() => invalidOptions(evalCase.id)));
+      if (info.type !== "Directory") {
+        return yield* invalidOptions(evalCase.id);
+      }
+      auth = {
+        mode: "native",
+        provider: authOption.provider,
+        agentDirectory: authOption.agentDirectory,
+      };
+    }
+
+    return {
+      runId,
+      repetition: options.repetition,
+      availableTools: [...CANONICAL_ALLOWED_TOOLS],
+      runtimeDirectory: options.runtimeDirectory,
+      auth,
+      model,
+      modelIdentity: `${auth.provider}/${model}`,
+      reasoning: options.reasoning,
+      mcpAuthorization,
+      timeoutMs,
+      serverUrl,
+      ...(options.sandbox === undefined ? {} : { sandbox: options.sandbox }),
+    };
   });
-};
 
 const listAllMcpTools = (
   client: MCPClient,
@@ -1438,24 +1509,22 @@ export const runOmpHarnessPluginEvalTrial = Function.dual<
                         },
                       ),
                   };
+                  const { auth } = validated;
                   const harness = createACP({
                     harnessId: "omp-acp",
                     builtinTools: OMP_BUILTIN_TOOLS,
                     source: {
                       type: "install-command",
-                      command: installCommand(
-                        validated.provider,
-                        validated.model,
-                        validated.providerBaseUrl,
-                        validated.runtimeDirectory,
-                      ),
+                      command: installCommand(validated.runtimeDirectory, auth, validated.model),
                     },
                     executable: "omp",
                     args: [
                       "acp",
                       "--no-extensions",
+                      "--config",
+                      `${validated.runtimeDirectory}/config.yml`,
                       "--provider",
-                      OMP_EVAL_PROVIDER_ALIAS,
+                      auth.mode === "api-key" ? OMP_EVAL_PROVIDER_ALIAS : auth.provider,
                       "--model",
                       validated.model,
                       "--thinking",
@@ -1464,14 +1533,20 @@ export const runOmpHarnessPluginEvalTrial = Function.dual<
                       "yolo",
                       "--no-session",
                     ],
-                    skillsDirectory: ".omp/agent/skills",
+                    skillsDirectory: ".agents/skills",
                     modelMapping: { type: "session-config-option", path: "model" },
                     mcpServers: {},
                     hostToolMcpTransport: "http",
-                    env: {
-                      NO_COLOR: "1",
-                      [OMP_EVAL_PROVIDER_API_KEY_ENV]: validated.apiKey,
-                    },
+                    env:
+                      auth.mode === "api-key"
+                        ? {
+                            NO_COLOR: "1",
+                            [OMP_EVAL_PROVIDER_API_KEY_ENV]: auth.apiKey,
+                          }
+                        : {
+                            NO_COLOR: "1",
+                            PI_CODING_AGENT_DIR: auth.agentDirectory,
+                          },
                   });
                   let sessionSkillsDirectory: string | undefined;
                   const agent = new HarnessAgent({
@@ -1483,7 +1558,7 @@ export const runOmpHarnessPluginEvalTrial = Function.dual<
                     sandboxConfig: {
                       onSession: ({ session, abortSignal }) =>
                         resolveSandboxHomeDir({ sandbox: session, abortSignal }).then((home) => {
-                          sessionSkillsDirectory = `${home}/.omp/agent/skills`;
+                          sessionSkillsDirectory = `${home}/.agents/skills`;
                         }),
                     },
                   });

From b95511e03d6628687c0bee349bc0df9293bf0f4d Mon Sep 17 00:00:00 2001
From: Eric Juta <ericjohnjuta@gmail.com>
Date: Fri, 11 Sep 2026 02:37:52 +0000
Subject: [PATCH 2/4] docs(evals): explain native profile auth and ACP limits

Document the explicit agent-directory choice, normal native profile writes, nested auth API migration, and synthetic stored-credential proof. Link the upstream structured provider-error report without claiming OAuth or live subscription verification.
---
 README.md                |  20 +++++---
 packages/evals/README.md | 105 ++++++++++++++++++++++++++++-----------
 2 files changed, 88 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index b317760..bb63ca7 100644
--- a/README.md
+++ b/README.md
@@ -66,21 +66,27 @@ requires `ASK_GINA_ACCESS_TOKEN`. Responses and Codex also need `OPENAI_API_KEY`
 Codex adds `CODEX_EVAL_EXECUTABLE` and `CODEX_EVAL_EXECUTABLE_SHA256`; OpenRouter
 needs `OPENROUTER_API_KEY` plus required `--openrouter-endpoint`,
 `--expected-provider`, and `--max-cost-usd`; Claude needs `ANTHROPIC_API_KEY` and
-`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_API_KEY`,
-`OMP_EVAL_EXECUTABLE`, `OMP_EVAL_EXECUTABLE_SHA256`, and
-`--provider openai|anthropic|openrouter`. Optional `--max-steps` applies only to
+`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_EXECUTABLE`,
+`OMP_EVAL_EXECUTABLE_SHA256`, and `--provider`. Its default API-key mode also
+requires `OMP_EVAL_API_KEY` and accepts `openai|anthropic|openrouter`.
+`--omp-auth native --omp-agent-dir <absolute-directory>` instead uses an
+explicit existing OMP profile and its native provider identifier. OMP can
+update that profile, including credential refreshes and database writes.
+Optional `--max-steps` applies only to
 OpenRouter and `--max-turns` only to Claude. Both default to 8 and accept 1 to 32.
 OpenRouter also accepts optional `--server-url` with the exact Gina-read URLs
 `https://askgina.ai/ai/gina/mcp` (production default) or
 `https://alpha.askgina.ai/ai/gina/mcp`. Other runners reject those OpenRouter-only
-flags. OMP has no step or turn flag. Native Codex, Claude, and OMP paths use
-explicit API keys, not a saved personal login. That existing API-key CLI does
-not meet a separately requested HarnessAgent or Codex saved-login policy.
+flags. OMP has no step or turn flag. Codex and Claude still use explicit API
+keys. OMP native mode passed a synthetic stored-credential smoke, not a real
+OAuth or subscription authentication test.
 OpenRouter uses local AI SDK MCP; Responses uses
 OpenAI-hosted MCP. Neither proves native plugin activation. Codex and Claude
 adapters distinguish native skill events from task conformance. Claude's live
 plugin activation remains unverified; offline fixtures and OMP local-runtime
 proof are not measured native-agent evidence. Stock OMP host-tool success and
 MCP-error handling passed a cold local synthetic smoke. Native ACP still does
-not classify provider HTTP failures. Flags, capture, and publication rules are
+not classify provider HTTP failures; this is tracked in
+[OMP #11644](https://github.com/can1357/oh-my-pi/issues/11644).
+Flags, capture, and publication rules are
 in `packages/evals/README.md`.
diff --git a/packages/evals/README.md b/packages/evals/README.md
index fc9957f..a7af0cb 100644
--- a/packages/evals/README.md
+++ b/packages/evals/README.md
@@ -21,6 +21,12 @@ CommonJS, browser and edge runtimes, and subpath imports are unsupported.
 consumers resolve the same adapter versions as the evaluators. There is no
 patched adapter, auth-store lease, or restricted-settings surface.
 
+`OmpHarnessTrialOptions` requires a nested `auth: OmpAuth` value. API-key
+callers pass `{ mode: "api-key", provider, apiKey }`, optionally with
+`providerBaseUrl`; native callers pass `{ mode: "native", provider, agentDirectory }`. The former
+top-level `provider`, `apiKey`, and `providerBaseUrl` fields are removed.
+The API-key provider type is `OmpApiKeyProvider`.
+
 `createLocalHarnessSandbox` is a synchronous local-process sandbox provider for
 stock `HarnessAgent`/`createACP` on POSIX hosts. Sessions run commands as
 ordinary host child processes in their own process group under a disposable
@@ -108,24 +114,32 @@ credentials differ:
 | OpenRouter    | `OPENROUTER_API_KEY`                                                                                                   |
 | Codex CLI     | `OPENAI_API_KEY`, absolute `CODEX_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `CODEX_EVAL_EXECUTABLE_SHA256` |
 | Claude CLI    | `ANTHROPIC_API_KEY` and absolute `CLAUDE_EVAL_EXECUTABLE`                                                              |
-| OMP harness   | `OMP_EVAL_API_KEY`, absolute `OMP_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256`   |
-
-The supported native Codex, Claude, and OMP paths use explicit API keys in
-disposable evaluation homes. They do not reuse a saved personal login. That
-existing API-key CLI does not meet a separately requested HarnessAgent or Codex
-saved-login policy. Saved-login reuse, refresh ownership, native OAuth
-subscriptions, and personal-home integration remain deferred. OMP does
-not read `~/.omp` or inherit `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` /
-`OPENROUTER_API_KEY`. Native OMP also never exports those names into the
-child. The child receives only `OMP_EVAL_PROVIDER_API_KEY` for
-the private `omp-eval` provider.
+| OMP API key   | `OMP_EVAL_API_KEY`, absolute `OMP_EVAL_EXECUTABLE`, and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256`   |
+| OMP native    | Absolute `OMP_EVAL_EXECUTABLE` and its lowercase SHA-256 digest in `OMP_EVAL_EXECUTABLE_SHA256`                        |
+
+Codex and Claude use explicit API keys in disposable evaluation homes, not a
+saved personal login. OMP defaults to `--omp-auth api-key`, which passes the
+model key as `OMP_EVAL_PROVIDER_API_KEY` for a private `omp-eval` provider.
+
+OMP also supports `--omp-auth native --omp-agent-dir <absolute-directory>`.
+The directory must already exist. OMP uses it directly through
+`PI_CODING_AGENT_DIR`; the evaluator does not copy credentials, generate native
+provider settings, or require `OMP_EVAL_API_KEY`. OMP may refresh credentials,
+update its databases, or migrate profile settings there. This is not a
+read-only profile mount. Choose the directory deliberately.
+
+Both OMP modes retain a minimal child environment and disable extensions.
+They do not inherit `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or
+`OPENROUTER_API_KEY`. Native mode relies on the selected profile's credential
+configuration, not arbitrary credential variables in the parent shell.
 
 Responses and Codex accept model IDs understood by their OpenAI backends, such as
 `gpt-5.1`. OpenRouter uses its `provider/model` namespace, such as
 `openai/gpt-5.1`. Claude uses a Claude CLI model ID or alias without the
-`anthropic/` prefix, such as `claude-sonnet-4-5-20250929`. OMP requires `--provider`
-to select `openai`, `anthropic`, or `openrouter`; `--model` uses that provider's
-backend ID. For example, `--provider openai --model gpt-5.1` records
+`anthropic/` prefix, such as `claude-sonnet-4-5-20250929`. OMP API-key mode requires
+`--provider openai|anthropic|openrouter`; native mode accepts a native OMP provider
+identifier, such as `openai-codex`. `--model` must be available through the
+selected provider. For example, `--provider openai --model gpt-5.1` records
 `openai/gpt-5.1`, while `--provider openrouter --model openai/gpt-5.1` records
 `openrouter/openai/gpt-5.1`. The report, attempt input, and observation keep that
 same identity. There is no separate displayed model. These examples show the
@@ -189,6 +203,19 @@ bun run eval:omp -- \
   --repetitions 3 \
   --account-class eval \
   --timeout-ms 120000
+
+bun run eval:omp -- \
+  --suite packages/evals/src/fixtures/ask-gina-routing-smoke.yaml \
+  --run-id local-omp-native-example \
+  --candidate main \
+  --omp-auth native \
+  --omp-agent-dir /absolute/path/to/existing/omp-agent \
+  --provider openai-codex \
+  --model gpt-5.6-luna \
+  --reasoning medium \
+  --repetitions 3 \
+  --account-class eval \
+  --timeout-ms 120000
 ```
 
 Repeat `--case <case-id>` to run a strict subset. `--timeout-ms` is required for
@@ -205,8 +232,10 @@ and accepts only the exact Gina-read URLs `https://askgina.ai/ai/gina/mcp`
 slashes and any other URL, including venue MCP endpoints, are rejected.
 `--max-steps` is optional only for OpenRouter, and `--max-turns` is optional
 only for Claude. Both default to `8` and accept `1` to `32`. `--provider` is
-required only for OMP. Other runners reject those OpenRouter, Claude, and OMP
-flags. OMP rejects `--max-steps` and `--max-turns`. ACP does not expose a
+required only for OMP. `--omp-auth` and `--omp-agent-dir` are also OMP-only.
+The directory flag is required for native mode and rejected in API-key mode.
+Other runners reject those OpenRouter, Claude, and OMP flags. OMP rejects
+`--max-steps` and `--max-turns`. ACP does not expose a
 portable native model-step boundary, so OMP does not claim an equal step budget
 with OpenRouter or Claude. Secrets have no command-line flags.
 A missing required flag or backend credential fails closed. The CLI does not
@@ -255,25 +284,33 @@ platforms fail closed.
 
 OMP requires the tested `18.1.17` executable. The CLI verifies its SHA-256 and
 snapshots it once, then starts a fresh local `omp acp` session per trial under
-`createLocalHarnessSandbox`. Bootstrap links that snapshot and writes a
-session-only `config.yml` and `models.yml` into a disposable home. There is no
-Docker engine, container image, or host-socket requirement.
-
-Each trial writes a static `models.yml` with one private `omp-eval` provider
-and one selected-model entry. Public `--provider` / `--model` identity stays
-on the report, attempt, and observation. Native ACP uses `--provider omp-eval`
+`createLocalHarnessSandbox`. Bootstrap links that snapshot. Both authentication
+modes load the isolated evaluation settings through an explicit `--config`
+overlay. There is no Docker engine, container image, or host-socket requirement.
+
+Each API-key trial writes a static `models.yml` in its disposable home with one
+private `omp-eval` provider and one selected-model entry. Public `--provider` /
+`--model` identity stays on the report, attempt, and observation. In this mode,
+ACP uses `--provider omp-eval`
 and the original backend model id, including OpenRouter nested slugs. The
 child receives the explicit model key as `OMP_EVAL_PROVIDER_API_KEY` through
 the stock adapter's environment option. Standard provider credential variables
 are not inherited. There is no credential broker or request transformation.
 
-OpenAI and OpenRouter use Chat Completions, not the Responses transport that
-OMP can select for built-in providers. Anthropic uses its Messages API. The
-CLI's `--reasoning` value selects native `--thinking`, with explicit effort
+In API-key mode, OpenAI and OpenRouter use Chat Completions, not the Responses
+transport that OMP can select for built-in providers. Anthropic uses its
+Messages API. The CLI's `--reasoning` value selects native `--thinking`, with explicit effort
 settings for OpenAI/OpenRouter and thinking budgets for Anthropic. The model
 entry leaves capacities, cost, and input modalities to OMP 18.1.17's bundled
 same-id metadata or defaults for unknown model ids.
 
+Native mode passes the requested provider unchanged and uses the selected
+profile's models and credentials. It writes no replacement `models.yml` and
+passes no evaluator model key. Its agent-directory check reads filesystem
+metadata only and rejects missing directories before MCP or model dispatch.
+Canonical skills are staged under the disposable `HOME/.agents/skills` in both
+modes, so selecting another agent directory does not redirect skill discovery.
+
 The Gina bearer stays on the host. Canonical MCP reads execute in the evaluator
 process and are exposed as wrapped HarnessAgent host tools. The child does not
 receive that token. Native OMP builtins still run with ordinary host access;
@@ -303,10 +340,18 @@ sessions. The stock Codex adapter also passed a cold local host-tool smoke.
 
 Native provider HTTP 500 errors arrive as ACP text plus `end_turn` with no typed
 metadata, so OMP cannot yet replace every evaluator for reliable provider-failure
-classification. Native OAuth subscriptions remain unproven. These synthetic
-model/MCP fixtures exercise the real native processes and protocols; they do
-not prove real model behavior, production Gina connectivity, or measured
-native-plugin activation.
+classification. This is tracked in
+[OMP #11644](https://github.com/can1357/oh-my-pi/issues/11644).
+
+Native mode passed a cold synthetic smoke with an inert API key stored through
+OMP's stock credential API in a disposable `agent.db`, with no evaluator key
+environment variable. The proof covered canonical skill reads, successful and
+failed MCP calls, rejection before dispatch for a missing profile, API-key mode
+compatibility, and one physical destroy per session. It left the selected
+profile in place; its database changed during native execution. This proves
+stored-credential delegation, not real OAuth refresh or subscription access.
+These fixtures do not prove real model behavior, production Gina connectivity,
+or measured native-plugin activation.
 
 Each live run writes a v1 aggregate below the ignored `.plugin-eval-runs/`
 directory and a sibling `.journal-v1.jsonl`, both mode `0600`. OpenRouter

From 3e2016414425b34beba443f7faa703242f020233 Mon Sep 17 00:00:00 2001
From: Eric Juta <ericjohnjuta@gmail.com>
Date: Fri, 11 Sep 2026 07:12:31 +0000
Subject: [PATCH 3/4] docs(evals): clarify native profile proof boundaries

Disambiguate the API-key ACP provider alias and record the actual nested profile hash evidence. Preserve the synthetic stored-key versus OAuth distinction and document that the selected profile database is writable.
---
 packages/evals/README.md | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/evals/README.md b/packages/evals/README.md
index a7af0cb..9b49495 100644
--- a/packages/evals/README.md
+++ b/packages/evals/README.md
@@ -290,9 +290,9 @@ overlay. There is no Docker engine, container image, or host-socket requirement.
 
 Each API-key trial writes a static `models.yml` in its disposable home with one
 private `omp-eval` provider and one selected-model entry. Public `--provider` /
-`--model` identity stays on the report, attempt, and observation. In this mode,
-ACP uses `--provider omp-eval`
-and the original backend model id, including OpenRouter nested slugs. The
+`--model` identity stays on the report, attempt, and observation. In API-key mode,
+the ACP child is launched with `--provider omp-eval` and the original backend
+model id, including OpenRouter nested slugs. The
 child receives the explicit model key as `OMP_EVAL_PROVIDER_API_KEY` through
 the stock adapter's environment option. Standard provider credential variables
 are not inherited. There is no credential broker or request transformation.
@@ -347,9 +347,11 @@ Native mode passed a cold synthetic smoke with an inert API key stored through
 OMP's stock credential API in a disposable `agent.db`, with no evaluator key
 environment variable. The proof covered canonical skill reads, successful and
 failed MCP calls, rejection before dispatch for a missing profile, API-key mode
-compatibility, and one physical destroy per session. It left the selected
-profile in place; its database changed during native execution. This proves
-stored-credential delegation, not real OAuth refresh or subscription access.
+compatibility, and one physical destroy per session. The receipt's
+`profile.before` and `profile.after` hashes matched for `models.yml` and
+`config.yml`, but `agent.db` changed during native execution. The selected
+profile remained in place. This is not a read-only profile mount and proves
+stored API-key delegation only, not real OAuth refresh or subscription access.
 These fixtures do not prove real model behavior, production Gina connectivity,
 or measured native-plugin activation.
 

From 13bfb4b3d3d34c50706631c520a97b81a331a279 Mon Sep 17 00:00:00 2001
From: Eric Juta <ericjohnjuta@gmail.com>
Date: Fri, 11 Sep 2026 08:11:43 +0000
Subject: [PATCH 4/4] fix(evals): validate native profiles before catalog
 preflight

Reject relative, missing, and non-directory OMP profile paths before authenticated MCP catalog traffic. Keep the direct-library guard and add an entrypoint regression with a valid-profile network control.
---
 packages/evals/__tests__/live-cli.test.ts | 129 ++++++++++++++++++++++
 packages/evals/src/bin/live.ts            |  12 ++
 2 files changed, 141 insertions(+)

diff --git a/packages/evals/__tests__/live-cli.test.ts b/packages/evals/__tests__/live-cli.test.ts
index 4199d0a..0224fda 100644
--- a/packages/evals/__tests__/live-cli.test.ts
+++ b/packages/evals/__tests__/live-cli.test.ts
@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import * as BunPath from "@effect/platform-bun/BunPath";
 import * as BunServices from "@effect/platform-bun/BunServices";
 import { assert, describe, it } from "@effect/vitest";
@@ -845,6 +847,133 @@ describe("live eval CLI subprocess", () => {
       ),
     );
 
+    it.effect("rejects a missing native profile before authenticated MCP dispatch", () =>
+      Effect.scoped(
+        Effect.gen(function* () {
+          const fs = yield* FileSystem.FileSystem;
+          const path = yield* Path.Path;
+          const pathValue = yield* Config.string("PATH");
+          const root = yield* fs.makeTempDirectoryScoped({ prefix: "live-cli-native-profile-" });
+          const cwd = path.join(root, "source");
+          const profile = path.join(root, "profile");
+          const preload = path.join(root, "block-fetch.mjs");
+          const callsPath = path.join(root, "fetch-calls");
+          const liveCli = path.join(process.cwd(), "packages/evals/src/bin/live.ts");
+          const executablePath = yield* fs.realPath("/bin/echo");
+          const executableBytes = yield* fs.readFile(executablePath);
+          const environment = { PATH: pathValue, HOME: root, GIT_CONFIG_NOSYSTEM: "1" };
+          yield* fs.makeDirectory(cwd);
+          yield* fs.makeDirectory(profile);
+          yield* fs.makeDirectory(path.join(cwd, "plugins/ask-gina"), { recursive: true });
+          yield* fs.copy(
+            path.join(process.cwd(), "plugins/ask-gina/skills"),
+            path.join(cwd, "plugins/ask-gina/skills"),
+          );
+          yield* fs.copyFile(
+            path.join(process.cwd(), "packages/evals/src/fixtures/ask-gina-routing-smoke.yaml"),
+            path.join(cwd, "suite.yaml"),
+          );
+          yield* fs.writeFileString(path.join(cwd, ".gitignore"), ".plugin-eval-runs/\n");
+          yield* fs.writeFileString(
+            preload,
+            `import { appendFileSync } from "node:fs";
+const callsPath = new URL("./fetch-calls", import.meta.url);
+globalThis.fetch = async () => {
+  appendFileSync(callsPath, "x");
+  throw new Error("native profile fixture blocked fetch");
+};
+`,
+          );
+          // The real entrypoint requires a clean source tree before preparing the OMP runtime.
+          for (const args of [
+            ["init", "--quiet"],
+            ["add", "."],
+            [
+              "-c",
+              "user.name=Native profile fixture",
+              "-c",
+              "user.email=native-profile@example.invalid",
+              "-c",
+              "core.hooksPath=/dev/null",
+              "commit",
+              "--quiet",
+              "--no-gpg-sign",
+              "-m",
+              "fixture",
+            ],
+          ]) {
+            const git = yield* ChildProcess.make("git", args, {
+              cwd,
+              env: environment,
+              extendEnv: false,
+              stdin: "ignore",
+              stdout: "ignore",
+              stderr: "pipe",
+            });
+            const [stderr, exitCode] = yield* Effect.all(
+              [collectBoundedUtf8Output(git.stderr, 65_536), git.exitCode],
+              { concurrency: "unbounded" },
+            );
+            assert.strictEqual(exitCode, 0, stderr.text);
+          }
+          // Run the control first so an unrelated preflight failure cannot fake zero requests.
+          for (const [runId, agentDirectory, reachesMcp] of [
+            ["run-control", profile, true],
+            ["run-missing", path.join(root, "missing-profile"), false],
+          ] as const) {
+            yield* fs.writeFileString(callsPath, "");
+            const child = yield* ChildProcess.make(
+              "bun",
+              [
+                "--preload",
+                preload,
+                liveCli,
+                ...requiredFlags("omp", [
+                  "--omp-auth",
+                  "native",
+                  "--provider",
+                  "openai-codex",
+                  "--omp-agent-dir",
+                  agentDirectory,
+                ]).map((value) => (value === "run-1" ? runId : value)),
+              ],
+              {
+                cwd,
+                env: {
+                  ...environment,
+                  ASK_GINA_ACCESS_TOKEN: "synthetic-native-profile-token",
+                  OMP_EVAL_EXECUTABLE: executablePath,
+                  OMP_EVAL_EXECUTABLE_SHA256: createHash("sha256")
+                    .update(executableBytes)
+                    .digest("hex"),
+                },
+                extendEnv: false,
+                stdin: "ignore",
+                stdout: "pipe",
+                stderr: "pipe",
+              },
+            );
+            const [stdout, stderr, exitCode] = yield* Effect.all(
+              [
+                collectBoundedUtf8Output(child.stdout, 65_536),
+                collectBoundedUtf8Output(child.stderr, 65_536),
+                child.exitCode,
+              ],
+              { concurrency: "unbounded" },
+            );
+            const output = `${stdout.text}\n${stderr.text}`;
+            const fetchCalls = (yield* fs.readFileString(callsPath)).length;
+            assert.notStrictEqual(exitCode, 0, output);
+            if (reachesMcp) {
+              assert.isAbove(fetchCalls, 0, output);
+            } else {
+              assert.strictEqual(fetchCalls, 0, output);
+            }
+          }
+        }),
+      ),
+    );
+
     it.effect("rejects an unsupported OpenRouter server URL before credentials", () =>
       Effect.scoped(
         Effect.gen(function* () {
diff --git a/packages/evals/src/bin/live.ts b/packages/evals/src/bin/live.ts
index 8568650..61e07d9 100755
--- a/packages/evals/src/bin/live.ts
+++ b/packages/evals/src/bin/live.ts
@@ -1320,6 +1320,18 @@ const run = (options: LiveEvalCliOptions) =>
         if (credentials.runner !== "omp") {
           return yield* new LiveEvalCliError({ reason: "omp-preflight-failed" });
         }
+        if (options.auth.mode === "native") {
+          if (!path.isAbsolute(options.auth.agentDirectory)) {
+            return yield* new LiveEvalCliError({ reason: "omp-preflight-failed" });
+          }
+          const fs = yield* FileSystem.FileSystem;
+          const profile = yield* fs
+            .stat(options.auth.agentDirectory)
+            .pipe(Effect.mapError(() => new LiveEvalCliError({ reason: "omp-preflight-failed" })));
+          if (profile.type !== "Directory") {
+            return yield* new LiveEvalCliError({ reason: "omp-preflight-failed" });
+          }
+        }
         const prepared = yield* prepareOmpHarnessRuntime({
           root,
           executablePath: credentials.executablePath,