Capture private OMP eval chat transcripts

packages/evals/README.md

@@ -373,9 +373,39 @@ or measured native-plugin activation.
 Each live run writes a v1 aggregate below the ignored `.plugin-eval-runs/`
 directory and a sibling `.journal-v1.jsonl`, both mode `0600`. OpenRouter
 writes four siblings: the v1 aggregate `.json`, `.requested-routing-v1.json`,
-`.configuration-v2.json`, and `.journal-v1.jsonl`. Keep all four. Other live
-runners write only the aggregate and journal. `--attempts-output` remains an
-optional grader companion and does not replace those siblings.
+`.configuration-v2.json`, and `.journal-v1.jsonl`. Keep all four. OMP also writes
+`<report-stem>.transcripts-v1.jsonl` automatically, in both authentication modes.
+Other live runners write only the aggregate and journal. `--attempts-output`
+remains an optional grader companion and does not replace those siblings.
+
+The OMP transcript is a private, mode-`0600` JSONL file. It contains a run header,
+one transcript per trial, and a final `report-bound` record with the exact
+aggregate report SHA-256. Each transcript records the user prompt, ordered
+assistant text, tool calls and results, trial identity, and settled status.
+Reasoning and raw provider frames are excluded. Capture does not change grading
+or the v1 aggregate and public attempt schemas.
+
+Supplied credentials and recognizable credential text are redacted before
+persistence. Other private text, tool data, and local paths can remain: do not
+publish this file or import it into the public leaderboard. Each trial is capped
+at 512 messages and 1 MiB, with 64 KiB field limits. Omitted or shortened content
+sets `truncated: true`.
+
+The transcript header is fsynced before credentials load. Completed and partial
+trial transcripts are written after session cleanup, including on timeout or
+interruption; this is not a per-token crash-recovery log. A killed process can
+leave only earlier trial records. Persistence has a five-second callback budget.
+A capture failure stops the run before the next dispatch, even if the completed
+observation has a failed grading status. An existing execution error or
+interruption remains authoritative. The final report binding requires the
+complete planned trial set.
+
+Library callers opt in with `OmpHarnessTrialOptions.onTranscript`. The package
+exports `OmpTrialTranscript`, its schemas, and `createOmpTranscriptWriter` for
+private storage. The callback receives a separate `AbortSignal` for its
+persistence window. Pass that signal to asynchronous storage so a timed-out
+callback does not leave a pending writer. Without that callback, library trials
+retain the existing non-streaming generation path.
 
 The journal writes its run header, fsynced, before credentials load. The
 header captures suite, catalog, reasoning, account class, selected cases,

packages/evals/__tests__/live-cli.test.ts

@@ -24,6 +24,7 @@ import { makeLiveEvalConfigurationEvidence } from "../src/configuration";
 import { ALPHA_GINA_READ_SERVER_URL } from "../src/server-url";
 import type { LiveEvalConfigurationCaptureType } from "../src/configuration";
 import { DEFAULT_OPENROUTER_MAX_TOOL_CALLS } from "../src/openrouter";
+import { OMP_TRANSCRIPT_SCHEMA_VERSION } from "../src/omp-transcript";
 import type { SanitizedEvalRunReport } from "../src/report";
 import aggregateFixture from "../src/fixtures/sanitized-aggregate.json";
 import { SanitizedEvalAggregateSchema } from "../src/sanitize";
@@ -847,6 +848,174 @@ describe("live eval CLI subprocess", () => {
       ),
     );
 
+    it.effect("rejects preexisting and attempts-colliding OMP transcripts before credentials", () =>
+      Effect.scoped(
+        Effect.gen(function* () {
+          const fs = yield* FileSystem.FileSystem;
+          const path = yield* Path.Path;
+          const pathValue = yield* Config.string("PATH");
+          const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "live-cli-omp-transcript-" });
+          const liveCli = path.join(process.cwd(), "packages/evals/src/bin/live.ts");
+          const outputDirectory = path.join(cwd, ".plugin-eval-runs");
+          yield* fs.makeDirectory(outputDirectory);
+
+          const preexistingRunId = "run-preexisting";
+          const preexistingTranscript = path.join(
+            outputDirectory,
+            `omp_harness-cand-1-${preexistingRunId}.transcripts-v1.jsonl`,
+          );
+          yield* fs.writeFileString(preexistingTranscript, "foreign bytes\n", {
+            flag: "wx",
+            mode: 0o600,
+          });
+          const preexistingChild = yield* ChildProcess.make(
+            "bun",
+            [
+              liveCli,
+              ...requiredFlags("omp", ["--provider", "openai"]).map((value) =>
+                value === "run-1" ? preexistingRunId : value,
+              ),
+            ],
+            {
+              cwd,
+              env: { PATH: pathValue },
+              extendEnv: false,
+              stdin: "ignore",
+              stdout: "pipe",
+              stderr: "pipe",
+            },
+          );
+          const [preexistingStdout, preexistingStderr, preexistingExitCode] = yield* Effect.all(
+            [
+              collectBoundedUtf8Output(preexistingChild.stdout, 65_536),
+              collectBoundedUtf8Output(preexistingChild.stderr, 65_536),
+              preexistingChild.exitCode,
+            ],
+            { concurrency: "unbounded" },
+          );
+          const preexistingOutput = `${preexistingStdout.text}\n${preexistingStderr.text}`;
+          assert.notStrictEqual(preexistingExitCode, 0, preexistingOutput);
+          assert.include(preexistingOutput, "PublicEvalAttemptWriteError");
+          assert.notInclude(preexistingOutput, "ASK_GINA_ACCESS_TOKEN");
+          assert.notInclude(preexistingOutput, "OMP_EVAL_API_KEY");
+          assert.strictEqual(yield* fs.readFileString(preexistingTranscript), "foreign bytes\n");
+          assert.isFalse(
+            yield* fs.exists(
+              path.join(outputDirectory, `omp_harness-cand-1-${preexistingRunId}.journal-v1.jsonl`),
+            ),
+          );
+
+          const collisionRunId = "run-collision";
+          const collisionTranscript = path.join(
+            outputDirectory,
+            `omp_harness-cand-1-${collisionRunId}.transcripts-v1.jsonl`,
+          );
+          const collisionChild = yield* ChildProcess.make(
+            "bun",
+            [
+              liveCli,
+              ...requiredFlags("omp", [
+                "--provider",
+                "openai",
+                "--attempts-output",
+                collisionTranscript,
+              ]).map((value) => (value === "run-1" ? collisionRunId : value)),
+            ],
+            {
+              cwd,
+              env: { PATH: pathValue },
+              extendEnv: false,
+              stdin: "ignore",
+              stdout: "pipe",
+              stderr: "pipe",
+            },
+          );
+          const [collisionStdout, collisionStderr, collisionExitCode] = yield* Effect.all(
+            [
+              collectBoundedUtf8Output(collisionChild.stdout, 65_536),
+              collectBoundedUtf8Output(collisionChild.stderr, 65_536),
+              collisionChild.exitCode,
+            ],
+            { concurrency: "unbounded" },
+          );
+          const collisionOutput = `${collisionStdout.text}\n${collisionStderr.text}`;
+          assert.notStrictEqual(collisionExitCode, 0, collisionOutput);
+          assert.include(collisionOutput, "PublicEvalAttemptWriteError");
+          assert.notInclude(collisionOutput, "ASK_GINA_ACCESS_TOKEN");
+          assert.isFalse(yield* fs.exists(collisionTranscript));
+          assert.isFalse(
+            yield* fs.exists(
+              path.join(outputDirectory, `omp_harness-cand-1-${collisionRunId}.journal-v1.jsonl`),
+            ),
+          );
+        }),
+      ),
+    );
+
+    it.effect("reserves a transcript companion only for OMP before credential loading", () =>
+      Effect.scoped(
+        Effect.gen(function* () {
+          const fs = yield* FileSystem.FileSystem;
+          const path = yield* Path.Path;
+          const pathValue = yield* Config.string("PATH");
+          const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "live-cli-omp-only-" });
+          const liveCli = path.join(process.cwd(), "packages/evals/src/bin/live.ts");
+          yield* fs.copyFile(
+            path.join(process.cwd(), "packages/evals/src/fixtures/ask-gina-routing-smoke.yaml"),
+            path.join(cwd, "suite.yaml"),
+          );
+
+          for (const [runner, runId, extra] of [
+            ["omp", "run-omp-companion", ["--provider", "openai"]],
+            ["responses", "run-responses-no-companion", []],
+          ] as const) {
+            const child = yield* ChildProcess.make(
+              "bun",
+              [
+                liveCli,
+                ...requiredFlags(runner, extra).map((value) => (value === "run-1" ? runId : value)),
+              ],
+              {
+                cwd,
+                env: { PATH: pathValue },
+                extendEnv: false,
+                stdin: "ignore",
+                stdout: "pipe",
+                stderr: "pipe",
+              },
+            );
+            const [stdout, stderr, exitCode] = yield* Effect.all(
+              [
+                collectBoundedUtf8Output(child.stdout, 65_536),
+                collectBoundedUtf8Output(child.stderr, 65_536),
+                child.exitCode,
+              ],
+              { concurrency: "unbounded" },
+            );
+            const output = `${stdout.text}\n${stderr.text}`;
+            assert.notStrictEqual(exitCode, 0, output);
+            assert.include(output, "ASK_GINA_ACCESS_TOKEN");
+          }
+
+          const outputDirectory = path.join(cwd, ".plugin-eval-runs");
+          const ompTranscriptPath = path.join(
+            outputDirectory,
+            "omp_harness-cand-1-run-omp-companion.transcripts-v1.jsonl",
+          );
+          const responsesTranscriptPath = path.join(
+            outputDirectory,
+            "responses_api-cand-1-run-responses-no-companion.transcripts-v1.jsonl",
+          );
+          assert.isTrue(yield* fs.exists(ompTranscriptPath));
+          assert.include(
+            yield* fs.readFileString(ompTranscriptPath),
+            `"schemaVersion":"${OMP_TRANSCRIPT_SCHEMA_VERSION}"`,
+          );
+          assert.isFalse(yield* fs.exists(responsesTranscriptPath));
+        }),
+      ),
+    );
+
     it.effect("rejects a missing native profile before authenticated MCP dispatch", () =>
       Effect.scoped(
         Effect.gen(function* () {