feat(evals): add multi-provider and native-agent runners

README.md

@@ -59,3 +59,20 @@ four-skill candidate archive, and contract, package, target, and eval receipts
 under ignored `dist/`. `bun run verify:artifacts` performs clean
 tarball installs and runtime checks. Nothing here publishes, releases, deploys,
 submits, or calls production during pull-request CI.
+
+Live eval runners are `bun run eval:responses`, `eval:codex`, `eval:openrouter`,
+`eval:claude`, and `eval:omp`. Hermetic replay is `eval:replay`. Every live runner
+requires `ASK_GINA_ACCESS_TOKEN`. Responses and Codex also need `OPENAI_API_KEY`;
+Codex adds `CODEX_EVAL_EXECUTABLE` and `CODEX_EVAL_EXECUTABLE_SHA256`; OpenRouter
+needs `OPENROUTER_API_KEY`; Claude needs `ANTHROPIC_API_KEY` and
+`CLAUDE_EVAL_EXECUTABLE`; OMP needs `OMP_EVAL_API_KEY`,
+`OMP_EVAL_EXECUTABLE`, `OMP_EVAL_EXECUTABLE_SHA256`, a local Docker engine, and
+`--provider openai|anthropic|openrouter`. Optional `--max-steps` applies only to
+OpenRouter and `--max-turns` only to Claude. Both default to 8 and accept 1 to 32.
+OMP has no step or turn flag. Native Codex, Claude, and OMP paths use explicit API
+keys, not a saved personal login. OpenRouter uses local AI SDK MCP; Responses uses
+OpenAI-hosted MCP. Neither proves native plugin activation. Codex and Claude
+adapters distinguish native skill events from task conformance. Claude's live
+plugin activation remains unverified; offline fixtures and OMP Docker/runtime
+proof are not measured native-agent evidence. Flags, capture, and publication
+rules are in `packages/evals/README.md`.

apps/evals/__tests__/public-results.test.ts

@@ -97,6 +97,88 @@ describe("public artifact browser boundary", () => {
     });
   });
 
+  it("accepts nested OpenRouter model identities in publications and current-index summaries", () => {
+    const model = "openrouter/openai/gpt-5.1";
+    const current = publication(currentBytes);
+    if (current.content.kind !== "result") throw new Error("Expected a result publication fixture");
+    const entry = history.publications[0];
+    if (entry?.summary === null || entry?.summary === undefined) {
+      throw new Error("Expected a current index summary fixture");
+    }
+
+    expect(
+      parsePublicArtifact(
+        encode({
+          ...current,
+          content: {
+            ...current.content,
+            result: {
+              ...current.content.result,
+              configuration: { ...current.content.result.configuration, model },
+            },
+          },
+        }),
+      ).kind,
+    ).toBe("publication");
+    expect(
+      parsePublicArtifact(
+        encode({
+          ...history,
+          publications: [
+            { ...entry, summary: { ...entry.summary, model } },
+            ...history.publications.slice(1),
+          ],
+        }),
+      ).kind,
+    ).toBe("index");
+  });
+
+  it("rejects malformed model identities without relaxing generic identifiers", () => {
+    const current = publication(currentBytes);
+    if (current.content.kind !== "result") throw new Error("Expected a result publication fixture");
+    const entry = history.publications[0];
+    if (entry?.summary === null || entry?.summary === undefined) {
+      throw new Error("Expected a current index summary fixture");
+    }
+
+    const malformedModels = [
+      "openrouter//gpt-5.1",
+      "openrouter/../gpt-5.1",
+      `openrouter/${"a".repeat(118)}`,
+    ] as const;
+    for (const model of malformedModels) {
+      expect(
+        parsePublicArtifact(
+          encode({
+            ...current,
+            content: {
+              ...current.content,
+              result: {
+                ...current.content.result,
+                configuration: { ...current.content.result.configuration, model },
+              },
+            },
+          }),
+        ).kind,
+      ).toBe("unsupported");
+      expect(
+        parsePublicArtifact(
+          encode({
+            ...history,
+            publications: [
+              { ...entry, summary: { ...entry.summary, model } },
+              ...history.publications.slice(1),
+            ],
+          }),
+        ).kind,
+      ).toBe("unsupported");
+    }
+
+    expect(parsePublicArtifact(encode({ ...current, publicationId: "openai/gpt-5.1" })).kind).toBe(
+      "unsupported",
+    );
+  });
+
   it("rejects malformed UTF-8 and BOM-prefixed JSON rather than normalizing the input", () => {
     expect(
       parsePublicArtifact(new Uint8Array([0x7b, 0x22, 0xc0, 0xaf, 0x22, 0x7d]).buffer).kind,

apps/evals/src/lib/public-results.ts

@@ -10,6 +10,8 @@ export type ParsedPublicArtifact =
 
 const WHITESPACE_ONLY = /^\s*$/u;
 const IDENTIFIER = /^[A-Za-z0-9](?:[A-Za-z0-9._:-]{0,126}[A-Za-z0-9])?$/u;
+const MODEL_IDENTIFIER =
+  /^[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?(?:\/[A-Za-z0-9](?:[A-Za-z0-9._:-]*[A-Za-z0-9])?)*$/u;
 const SHA_256 = /^[a-f0-9]{64}$/u;
 const ATTEMPT_ID = /^attempt-[a-f0-9]{64}$/u;
 const UTC_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{3})?Z$/u;
@@ -39,6 +41,9 @@ const isText = (value: unknown, max = 128): value is string =>
 
 const isIdentifier = (value: unknown): value is string => isText(value) && IDENTIFIER.test(value);
 
+const isModelIdentifier = (value: unknown): value is string =>
+  isText(value) && MODEL_IDENTIFIER.test(value);
+
 const isSha256 = (value: unknown): value is string =>
   typeof value === "string" && SHA_256.test(value);
 
@@ -231,7 +236,7 @@ const hasResult = (value: unknown): value is PublicEvalResult => {
     isObject(configuration) &&
     (configuration.availability === "pinned" || configuration.availability === "labels_only") &&
     isIdentifier(configuration.candidate) &&
-    isIdentifier(configuration.model) &&
+    isModelIdentifier(configuration.model) &&
     (configuration.reasoning === null || isIdentifier(configuration.reasoning)) &&
     (configuration.pinnedSha256 === null || isSha256(configuration.pinnedSha256)) &&
     isObject(coverage) &&
@@ -386,7 +391,7 @@ const hasIndex = (value: unknown): value is PublicEvalIndex => {
           (isObject(summary) &&
             isIdentifier(summary.suiteId) &&
             isIdentifier(summary.candidate) &&
-            isIdentifier(summary.model) &&
+            isModelIdentifier(summary.model) &&
             isTimestamp(summary.startedAt))) &&
         Array.isArray(revisions) &&
         revisions.length > 0