Add live Codex lifecycle conformance tests

.github/workflows/integration-tests.yml

@@ -66,6 +66,7 @@ jobs:
           ANTHROPIC_TESTING_KEY: ${{ secrets.ANTHROPIC_TESTING_KEY }}
         run: |
           if [ -n "$ANTHROPIC_TESTING_KEY" ]; then
+            echo "::add-mask::$ANTHROPIC_TESTING_KEY"
             echo "available=true" >> "$GITHUB_OUTPUT"
           else
             echo "ANTHROPIC_TESTING_KEY is unavailable; skipping live Claude tests."
@@ -106,6 +107,60 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
+  codex-live:
+    name: Live Codex conformance
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Check for OpenAI testing key
+        id: openai-key
+        shell: bash
+        env:
+          OPENAI_TESTING_KEY: ${{ secrets.OPENAI_TESTING_KEY }}
+        run: |
+          if [ -n "$OPENAI_TESTING_KEY" ]; then
+            echo "::add-mask::$OPENAI_TESTING_KEY"
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "OPENAI_TESTING_KEY is unavailable; skipping live Codex tests."
+          fi
+
+      - name: Check out repository
+        if: steps.openai-key.outputs.available == 'true'
+        uses: actions/checkout@v4
+
+      - name: Set up Bun
+        if: steps.openai-key.outputs.available == 'true'
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Install server dependencies
+        if: steps.openai-key.outputs.available == 'true'
+        working-directory: server
+        run: bun install --frozen-lockfile
+
+      - name: Install integration-test dependencies
+        if: steps.openai-key.outputs.available == 'true'
+        working-directory: integration-tests
+        run: bun install --frozen-lockfile
+
+      - name: Run live Codex conformance tests
+        if: steps.openai-key.outputs.available == 'true'
+        env:
+          OPENAI_TESTING_KEY: ${{ secrets.OPENAI_TESTING_KEY }}
+        run: bun run test:live:codex
+
+      - name: Upload live Codex diagnostics
+        if: failure() && steps.openai-key.outputs.available == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: codex-live-integration-diagnostics
+          path: integration-tests/artifacts/server/
+          if-no-files-found: ignore
+          retention-days: 7
+
   lightpanda-e2e:
     name: Lightpanda E2E
     runs-on: ubuntu-latest

integration-tests/README.md

@@ -99,7 +99,7 @@ Lightpanda validates DOM, routing, events, and browser/server coordination. It d
 - Assert `fixture.assertNoBrowserErrors()` in successful E2E workflows. Do not swallow console, protocol, cleanup, or shutdown failures.
 - Do not commit generated files under `artifacts/`. Successful runs remove their temporary directories automatically.
 
-Set `KEEP_INTEGRATION_ARTIFACTS=1` to retain isolated fixture directories for investigation. Failed server tests write diagnostics under `artifacts/server/`; failed E2E tests write the DOM snapshot, browser errors, Lightpanda logs, server exchanges, WebSocket events, and provider requests under `artifacts/e2e/`.
+Set `KEEP_INTEGRATION_ARTIFACTS=1` to retain isolated non-credential fixture directories for investigation. Credential-backed fixtures always remove their temporary provider homes and transcripts. Failed server tests write diagnostics under `artifacts/server/`; failed E2E tests write the DOM snapshot, browser errors, Lightpanda logs, server exchanges, WebSocket events, and provider requests under `artifacts/e2e/`.
 
 ## Running Tests
 
@@ -110,6 +110,7 @@ bun run typecheck
 bun run test:integration:server
 
 ANTHROPIC_TESTING_KEY=... bun run test:live:claude
+OPENAI_TESTING_KEY=... bun run test:live:codex
 
 bun run build
 LIGHTPANDA_BIN=/path/to/lightpanda bun run test:integration:e2e
@@ -118,7 +119,7 @@ bun run check
 bun run test
 ```
 
-`bun run test:integration` runs the deterministic server integration lane but not the credential-backed live-provider or Lightpanda lanes. The root `bun run test` command runs the server and web unit suites, so run the integration commands explicitly while developing cross-boundary changes. Credential-backed suites use the separate `test:live` and `test:live:<provider>` commands; do not run them locally unless actively changing those tests, and rely on the PR CI live-provider gate otherwise. The live Claude lane requires `ANTHROPIC_TESTING_KEY`; it uses the pinned test-only Claude CLI, Haiku, low effort, a temporary Claude home without changing the user's CLI login, and redacted failure diagnostics that omit provider content and server logs. The E2E fixture requires a current production build at `web/build/index.html` and an executable `LIGHTPANDA_BIN`. CI pins and verifies the Lightpanda binary in `.github/workflows/integration-tests.yml`.
+`bun run test:integration` runs the deterministic server integration lane but not the credential-backed live-provider or Lightpanda lanes. The root `bun run test` command runs the server and web unit suites, so run the integration commands explicitly while developing cross-boundary changes. Credential-backed suites use the separate `test:live` and `test:live:<provider>` commands; do not run them locally unless actively changing those tests, and rely on the PR CI live-provider gate otherwise. The live Claude lane requires `ANTHROPIC_TESTING_KEY` and uses the pinned test-only Claude CLI, Haiku, and low effort. The live Codex lane requires `OPENAI_TESTING_KEY` and uses the pinned test-only Codex CLI, `gpt-5.4-nano`, and low effort. Both live lanes use a temporary provider home without changing the user's CLI login and redact failure diagnostics that omit provider content and server logs. The E2E fixture requires a current production build at `web/build/index.html` and an executable `LIGHTPANDA_BIN`. CI pins and verifies the Lightpanda binary in `.github/workflows/integration-tests.yml`.
 
 Focused runs are useful while iterating:
 

integration-tests/package.json

@@ -6,12 +6,14 @@
     "test": "bun run test:server",
     "test:server": "bun test --max-concurrency=1 --timeout=30000 tests/server",
     "test:e2e": "bun test --max-concurrency=1 --timeout=60000 tests/e2e",
-    "test:live": "bun run test:live:claude",
+    "test:live": "bun run test:live:claude && bun run test:live:codex",
     "test:live:claude": "bun test --max-concurrency=1 --timeout=120000 tests/live/claude",
+    "test:live:codex": "bun test --max-concurrency=1 --timeout=240000 tests/live/codex",
     "typecheck": "bunx tsc -p tsconfig.json"
   },
   "devDependencies": {
     "@anthropic-ai/claude-code": "2.1.220",
+    "@openai/codex": "0.144.6",
     "@types/bun": "1.3.14",
     "puppeteer-core": "25.3.0",
     "typescript": "7.0.2"

integration-tests/support/garcon-process.ts

@@ -12,9 +12,26 @@ export interface GarconProcessOptions {
   homeDir: string;
   startupTimeoutMs?: number;
   environment?: Record<string, string>;
+  redactEnvironmentValues?: boolean;
 }
 
 type GarconChild = Bun.Subprocess<'ignore', 'pipe', 'pipe'>;
+const SENSITIVE_ENVIRONMENT_NAME =
+  /(?:api[_-]?key|auth[_-]?token|credential|password|secret|token)/i;
+
+export function redactSensitiveEnvironmentText(
+  text: string,
+  environment: Record<string, string> = {},
+): string {
+  const values = Object.entries(environment)
+    .filter(([name, value]) => SENSITIVE_ENVIRONMENT_NAME.test(name) && value.length > 0)
+    .map(([, value]) => value)
+    .sort((left, right) => right.length - left.length);
+  return values.reduce(
+    (redacted, value) => redacted.replaceAll(value, '[REDACTED]'),
+    text,
+  );
+}
 
 function isolatedEnvironment(
   homeDir: string,
@@ -75,16 +92,23 @@ export class GarconProcess {
   #unexpectedExit: string | null = null;
   #exitCode: number | null = null;
 
-  private constructor(child: GarconChild, ready: Deferred<string>) {
+  private constructor(
+    child: GarconChild,
+    ready: Deferred<string>,
+    redactedEnvironment: Record<string, string>,
+  ) {
     this.#child = child;
     let readinessText = '';
     const inspectText = (text: string) => {
       readinessText = `${readinessText}${text}`.slice(-2_000);
       const match = SERVER_READY_PATTERN.exec(readinessText);
       if (match) ready.resolve(match[1]);
     };
-    this.#stdoutPump = pumpLines(child.stdout, 'stdout', inspectText, (line) => this.#logs.push(line));
-    this.#stderrPump = pumpLines(child.stderr, 'stderr', inspectText, (line) => this.#logs.push(line));
+    const captureLine = (line: string) => {
+      this.#logs.push(redactSensitiveEnvironmentText(line, redactedEnvironment));
+    };
+    this.#stdoutPump = pumpLines(child.stdout, 'stdout', inspectText, captureLine);
+    this.#stderrPump = pumpLines(child.stderr, 'stderr', inspectText, captureLine);
     void child.exited.then((exitCode) => {
       this.#exitCode = exitCode;
       if (!this.#expectedExit) {
@@ -118,7 +142,11 @@ export class GarconProcess {
       stdout: 'pipe',
       stderr: 'pipe',
     });
-    const instance = new GarconProcess(child, ready);
+    const instance = new GarconProcess(
+      child,
+      ready,
+      options.redactEnvironmentValues ? options.environment ?? {} : {},
+    );
 
     try {
       instance.#baseUrl = await withTimeout(

integration-tests/support/integration-fixture.ts

@@ -171,6 +171,7 @@ export class IntegrationFixture {
         projectDir: dirs.project,
         homeDir: dirs.home,
         environment: options.serverEnvironment,
+        redactEnvironmentValues: options.redactSensitiveDiagnostics,
       });
       client = await GarconTestClient.connect(garcon.baseUrl, {
         redactSensitiveDiagnostics: options.redactSensitiveDiagnostics,
@@ -431,7 +432,10 @@ export class IntegrationFixture {
     this.fakeProviders.openAiResponses.stop();
     this.fakeProviders.anthropic.stop();
 
-    if (errors.length === 0 && process.env.KEEP_INTEGRATION_ARTIFACTS !== '1') {
+    if (
+      this.#redactSensitiveDiagnostics
+      || (errors.length === 0 && process.env.KEEP_INTEGRATION_ARTIFACTS !== '1')
+    ) {
       await rm(this.dirs.root, { recursive: true, force: true });
     }
     if (errors.length > 0) {
@@ -450,6 +454,7 @@ export class IntegrationFixture {
       projectDir: this.dirs.project,
       homeDir: this.dirs.home,
       environment: this.#serverEnvironment,
+      redactEnvironmentValues: this.#redactSensitiveDiagnostics,
     });
     this.client = await GarconTestClient.connect(this.garcon.baseUrl, {
       redactSensitiveDiagnostics: this.#redactSensitiveDiagnostics,

integration-tests/support/live-agent.ts

@@ -0,0 +1,76 @@
+import { expect } from 'bun:test';
+import type { ServerWsMessage } from '../../common/ws-events.js';
+import type { IntegrationFixture } from './integration-fixture.js';
+
+export const LIVE_TURN_TIMEOUT_MS = 90_000;
+
+export function liveMarker(label: string): string {
+  return `GARCON_LIVE_${label}_${crypto.randomUUID().replaceAll('-', '')}`;
+}
+
+export function exactReplyPrompt(value: string): string {
+  return `Reply with exactly ${value}. Do not use tools.`;
+}
+
+export function expectFinished(type: string): void {
+  expect(type).toBe('agent-run-finished');
+}
+
+export function expectAssistantMarker(contents: readonly string[], value: string): void {
+  expect(contents.some((content) => content.includes(value))).toBe(true);
+}
+
+function expectVisibleResponseBeforeSettlement(input: {
+  events: readonly ServerWsMessage[];
+  chatId: string;
+  turnId: string | undefined;
+  marker: string;
+}): void {
+  const processingStarted = input.events.findIndex((event) =>
+    event.type === 'chat-processing-updated'
+    && event.chatId === input.chatId
+    && event.isProcessing);
+  const assistantResponse = input.events.findIndex((event) =>
+    event.type === 'chat-messages'
+    && event.chatId === input.chatId
+    && event.messages.some((entry) =>
+      entry.message.type === 'assistant-message'
+      && entry.message.content.includes(input.marker)));
+  const processingStopped = input.events.findIndex((event) =>
+    event.type === 'chat-processing-updated'
+    && event.chatId === input.chatId
+    && !event.isProcessing);
+  const terminal = input.events.findIndex((event) =>
+    (event.type === 'agent-run-finished' || event.type === 'agent-run-failed')
+    && event.chatId === input.chatId
+    && event.turnId === input.turnId);
+
+  expect(processingStarted).toBeGreaterThanOrEqual(0);
+  expect(assistantResponse).toBeGreaterThan(processingStarted);
+  expect(processingStopped).toBeGreaterThan(assistantResponse);
+  expect(terminal).toBeGreaterThan(assistantResponse);
+}
+
+export async function waitForVisibleResponse(input: {
+  fixture: IntegrationFixture;
+  chatId: string;
+  turnId: string | undefined;
+  marker: string;
+  afterIndex: number;
+}): Promise<void> {
+  expectFinished((await input.fixture.client.waitForTurnTerminal(
+    input.chatId,
+    input.turnId,
+    { afterIndex: input.afterIndex, timeoutMs: LIVE_TURN_TIMEOUT_MS },
+  )).type);
+  await input.fixture.client.waitForProcessing(input.chatId, false, {
+    afterIndex: input.afterIndex,
+    timeoutMs: LIVE_TURN_TIMEOUT_MS,
+  });
+  expectVisibleResponseBeforeSettlement({
+    events: input.fixture.client.eventsSince(input.afterIndex),
+    chatId: input.chatId,
+    turnId: input.turnId,
+    marker: input.marker,
+  });
+}