Add live Claude lifecycle conformance tests (#408)

* docs: define live agent test guardrails

* test(integration): protect credential-backed diagnostics

* test(claude): add live lifecycle conformance gate

Author: cfal

.github/workflows/integration-tests.yml

@@ -53,6 +53,59 @@ jobs:
           if-no-files-found: ignore
           retention-days: 7
 
+  claude-live:
+    name: Live Claude conformance
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Check for Anthropic testing key
+        id: anthropic-key
+        shell: bash
+        env:
+          ANTHROPIC_TESTING_KEY: ${{ secrets.ANTHROPIC_TESTING_KEY }}
+        run: |
+          if [ -n "$ANTHROPIC_TESTING_KEY" ]; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "ANTHROPIC_TESTING_KEY is unavailable; skipping live Claude tests."
+          fi
+
+      - name: Check out repository
+        if: steps.anthropic-key.outputs.available == 'true'
+        uses: actions/checkout@v4
+
+      - name: Set up Bun
+        if: steps.anthropic-key.outputs.available == 'true'
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Install server dependencies
+        if: steps.anthropic-key.outputs.available == 'true'
+        working-directory: server
+        run: bun install --frozen-lockfile
+
+      - name: Install integration-test dependencies
+        if: steps.anthropic-key.outputs.available == 'true'
+        working-directory: integration-tests
+        run: bun install --frozen-lockfile
+
+      - name: Run live Claude conformance tests
+        if: steps.anthropic-key.outputs.available == 'true'
+        env:
+          ANTHROPIC_TESTING_KEY: ${{ secrets.ANTHROPIC_TESTING_KEY }}
+        run: bun run test:live:claude
+
+      - name: Upload live Claude diagnostics
+        if: failure() && steps.anthropic-key.outputs.available == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: claude-live-integration-diagnostics
+          path: integration-tests/artifacts/server/
+          if-no-files-found: ignore
+          retention-days: 7
+
   lightpanda-e2e:
     name: Lightpanda E2E
     runs-on: ubuntu-latest

AGENTS.md

@@ -389,6 +389,11 @@ After completion of a task, verify:
 
 Integration coverage is mandatory when correctness crosses server, HTTP/WebSocket, persistence, provider, or SPA boundaries. Add black-box server tests under `integration-tests/tests/server` for chat lifecycle, queueing, reconnect, restart semantics (including empty execution state), provider failure, ownership-journal recovery, fork, and deletion behavior. Add Lightpanda tests under `integration-tests/tests/e2e` when the browser workflow itself is part of the contract. Every production regression in these flows must gain an integration test that reproduces it; unit tests remain required for the underlying component behavior.
 
+- Always use Haiku for Claude integration tests and `gpt-5.4-nano` for Codex integration tests.
+- Always use the lowest supported reasoning effort in integration tests.
+- Keep credential-backed agent suites under `test:live:*`, outside routine test commands.
+- Never run live-agent tests locally unless actively changing those tests; rely on the PR CI live-provider gate otherwise.
+
 ### Regression Focus Areas
 
 - Chat lifecycle transitions.

integration-tests/README.md

@@ -109,14 +109,16 @@ From the repository root:
 bun run typecheck
 bun run test:integration:server
 
+ANTHROPIC_TESTING_KEY=... bun run test:live:claude
+
 bun run build
 LIGHTPANDA_BIN=/path/to/lightpanda bun run test:integration:e2e
 
 bun run check
 bun run test
 ```
 
-`bun run test:integration` runs the server integration lane but not the Lightpanda lane. The root `bun run test` command runs the server and web unit suites, so run the integration commands explicitly while developing cross-boundary changes. The E2E fixture requires a current production build at `web/build/index.html` and an executable `LIGHTPANDA_BIN`. CI pins and verifies the Lightpanda binary in `.github/workflows/integration-tests.yml`.
+`bun run test:integration` runs the deterministic server integration lane but not the credential-backed live-provider or Lightpanda lanes. The root `bun run test` command runs the server and web unit suites, so run the integration commands explicitly while developing cross-boundary changes. Credential-backed suites use the separate `test:live` and `test:live:<provider>` commands; do not run them locally unless actively changing those tests, and rely on the PR CI live-provider gate otherwise. The live Claude lane requires `ANTHROPIC_TESTING_KEY`; it uses the pinned test-only Claude CLI, Haiku, low effort, a temporary Claude home without changing the user's CLI login, and redacted failure diagnostics that omit provider content and server logs. The E2E fixture requires a current production build at `web/build/index.html` and an executable `LIGHTPANDA_BIN`. CI pins and verifies the Lightpanda binary in `.github/workflows/integration-tests.yml`.
 
 Focused runs are useful while iterating:
 
@@ -132,7 +134,9 @@ Keep the configured single-test concurrency. The suites exercise process lifecyc
 
 The deterministic lane exercises `direct-openai-compatible` and `direct-anthropic-compatible` through separate protocol fakes. Both providers are present in every fixture and remain alive across Garcon restarts. The Anthropic fake covers the Messages HTTP and SSE contract; it is not a fake Claude Code process or Claude Agent SDK. Together the fakes prove Garcon's multi-agent routing, lifecycle, queue, transcript, persistence, search, and SPA behavior without spending credentials or depending on external availability. They do not prove the native behavior of Claude, Codex, Pi, OpenCode, Factory, Amp, Cursor, or other provider binaries.
 
-Future credential-backed provider suites should be opt-in, isolated from the deterministic required lane, and explicit about cost, rate limits, cleanup, and supported environments. Add them when test API keys are available; do not weaken or replace fake-provider coverage with live-provider tests.
+Credential-backed provider suites remain isolated from the deterministic required lane and must be explicit about cost, rate limits, cleanup, and supported environments. The live Claude lane receives its testing key only in the temporary Garcon process environment. Do not weaken or replace fake-provider coverage with live-provider tests.
+
+The live Claude lane covers correlated activity, consecutive queue dispatch, whole and historical forks, immediate re-forks, native graph isolation, restart/resume, permission decisions, live and reloaded tool results, interruption, stop, and post-cancellation recovery.
 
 ## Change Checklist
 

integration-tests/bun.lock

@@ -6,9 +6,12 @@
     "test": "bun run test:server",
     "test:server": "bun test --max-concurrency=1 --timeout=30000 tests/server",
     "test:e2e": "bun test --max-concurrency=1 --timeout=60000 tests/e2e",
+    "test:live": "bun run test:live:claude",
+    "test:live:claude": "bun test --max-concurrency=1 --timeout=120000 tests/live/claude",
     "typecheck": "bunx tsc -p tsconfig.json"
   },
   "devDependencies": {
+    "@anthropic-ai/claude-code": "2.1.220",
     "@types/bun": "1.3.14",
     "puppeteer-core": "25.3.0",
     "typescript": "7.0.2"

integration-tests/package.json

@@ -19,6 +19,7 @@ import type {
   ForkChatResponse,
   ForkRunCommandRequest,
   ForkRunCommandResponse,
+  PermissionDecisionCommandRequest,
   QueueEntryCommandResponse,
   QueueEntryCreateCommandRequest,
   QueueEntryDeleteCommandRequest,
@@ -162,10 +163,24 @@ interface GarconWebSocket {
 
 export interface GarconTestClientOptions {
   createWebSocket?: (url: string) => GarconWebSocket;
+  redactSensitiveDiagnostics?: boolean;
 }
 
 const WEB_SOCKET_OPEN = 1;
 const WEB_SOCKET_CLOSED = 3;
+const SAFE_DIAGNOSTIC_STRING_KEYS = new Set([
+  'agentid',
+  'code',
+  'method',
+  'model',
+  'permissionmode',
+  'requesttype',
+  'status',
+  'stopintent',
+  'thinkingmode',
+  'timestamp',
+  'type',
+]);
 
 export class GarconApiError extends Error {
   constructor(
@@ -186,8 +201,20 @@ export class GarconWsRequestError extends Error {
   }
 }
 
-function redact(value: unknown): unknown {
-  if (Array.isArray(value)) return value.map(redact);
+function redact(
+  value: unknown,
+  redactSensitiveDiagnostics = false,
+  parentKey?: string,
+): unknown {
+  if (Array.isArray(value)) {
+    return value.map((item) => redact(item, redactSensitiveDiagnostics, parentKey));
+  }
+  if (redactSensitiveDiagnostics && typeof value === 'string') {
+    const normalized = parentKey?.toLowerCase() ?? '';
+    return SAFE_DIAGNOSTIC_STRING_KEYS.has(normalized) || normalized.endsWith('id')
+      ? value
+      : '[REDACTED]';
+  }
   if (!isRecord(value)) return value;
   return Object.fromEntries(Object.entries(value).map(([key, item]) => {
     const normalized = key.toLowerCase();
@@ -199,7 +226,7 @@ function redact(value: unknown): unknown {
     ) {
       return [key, '[REDACTED]'];
     }
-    return [key, redact(item)];
+    return [key, redact(item, redactSensitiveDiagnostics, key)];
   }));
 }
 
@@ -234,6 +261,7 @@ async function responseBody(response: Response): Promise<unknown> {
 export class GarconTestClient {
   readonly #baseUrl: string;
   readonly #createWebSocket: (url: string) => GarconWebSocket;
+  readonly #redactSensitiveDiagnostics: boolean;
   readonly #exchanges: HttpExchange[] = [];
   readonly #eventRecords: EventRecord[] = [];
   readonly #waiters = new Set<EventWaiter>();
@@ -244,6 +272,7 @@ export class GarconTestClient {
   private constructor(baseUrl: string, options: GarconTestClientOptions) {
     this.#baseUrl = baseUrl.replace(/\/$/, '');
     this.#createWebSocket = options.createWebSocket ?? ((url) => new WebSocket(url));
+    this.#redactSensitiveDiagnostics = options.redactSensitiveDiagnostics === true;
   }
 
   static async connect(baseUrl: string, options: GarconTestClientOptions = {}): Promise<GarconTestClient> {
@@ -273,7 +302,12 @@ export class GarconTestClient {
   }
 
   eventRecords(): readonly EventRecord[] {
-    return this.#eventRecords.slice();
+    if (!this.#redactSensitiveDiagnostics) return this.#eventRecords.slice();
+    return this.#eventRecords.map((record) => ({
+      raw: redact(record.raw, true),
+      parsed: redact(record.parsed, true) as ServerWsMessage,
+      receivedAt: record.receivedAt,
+    }));
   }
 
   exchanges(): readonly HttpExchange[] {
@@ -503,6 +537,15 @@ export class GarconTestClient {
     return this.post<ForkRunCommandResponse>('/api/v1/chats/fork-run', request);
   }
 
+  sendPermissionDecision(
+    request: PermissionDecisionCommandRequest,
+  ): Promise<CommandAcceptedResponse> {
+    return this.post<CommandAcceptedResponse>(
+      '/api/v1/chats/permissions/decision',
+      request,
+    );
+  }
+
   switchAgentModel(request: AgentModelPatchRequest): Promise<AgentModelPatchResponse> {
     return this.patch<AgentModelPatchResponse>('/api/v1/chats/agent-model', request);
   }
@@ -798,11 +841,11 @@ export class GarconTestClient {
   }
 
   describeEvents(): string {
-    return JSON.stringify(this.#eventRecords.map((record, index) => ({
+    return JSON.stringify(redact(this.#eventRecords.map((record, index) => ({
       index,
       receivedAt: record.receivedAt,
       event: record.parsed,
-    })), null, 2);
+    })), this.#redactSensitiveDiagnostics), null, 2);
   }
 
   private async request<T>(method: string, path: string, body?: unknown): Promise<T> {
@@ -816,8 +859,10 @@ export class GarconTestClient {
       method,
       path,
       status: response.status,
-      ...(body === undefined ? {} : { requestBody: redact(body) }),
-      responseBody: redact(parsed),
+      ...(body === undefined
+        ? {}
+        : { requestBody: redact(body, this.#redactSensitiveDiagnostics) }),
+      responseBody: redact(parsed, this.#redactSensitiveDiagnostics),
     });
     if (!response.ok) throw new GarconApiError(response.status, parsed, method, path);
     return parsed as T;