fix(cloud): harden mock stack e2e harness

Author: NubsCarson

packages/cloud-api/v1/_container-control-plane-forward.ts

@@ -51,15 +51,19 @@ async function forwardControlPlaneRequest(
   configureHeaders(headers);
 
   try {
-    const upstream = await fetch(target, {
-      body:
-        c.req.method === "GET" || c.req.method === "HEAD"
-          ? undefined
-          : c.req.raw.body,
+    const body =
+      c.req.method === "GET" || c.req.method === "HEAD"
+        ? undefined
+        : c.req.raw.body;
+    const init: RequestInit & { duplex?: "half" } = {
+      body,
       headers,
       method: c.req.method,
       redirect: "manual",
-    });
+    };
+    if (body) init.duplex = "half";
+
+    const upstream = await fetch(target, init);
 
     return new Response(upstream.body, {
       headers: upstream.headers,

packages/cloud-shared/src/lib/services/api-keys.ts

@@ -254,11 +254,10 @@ export class ApiKeysService {
 
   async revokeForAgent(agentSandboxId: string): Promise<void> {
     const name = ApiKeysService.agentApiKeyName(agentSandboxId);
-    const keys = await apiKeysRepository.findByName(name);
+    const keys = await apiKeysRepository.deleteByName(name);
     for (const key of keys) {
       await this.invalidateCache(key.key_hash);
     }
-    await apiKeysRepository.deleteByName(name);
   }
 }
 

packages/cloud-shared/src/lib/services/eliza-sandbox.ts

@@ -516,7 +516,7 @@ export class ElizaSandboxService {
   }
 
   async deleteAgent(agentId: string, orgId: string): Promise<DeleteAgentResult> {
-    return dbWrite.transaction(async (tx) => {
+    const result = await dbWrite.transaction(async (tx) => {
       await this.lockLifecycle(tx, agentId, orgId);
 
       const rec = await this.getAgentForLifecycleMutation(tx, agentId, orgId);
@@ -584,23 +584,26 @@ export class ElizaSandboxService {
       `);
       const deletedSandbox = result.rows[0];
 
-      if (deletedSandbox) {
-        // Best-effort: revoke the per-agent API key. A failure here doesn't
-        // un-delete the sandbox; the key just lingers as inactive data.
-        try {
-          await apiKeysService.revokeForAgent(agentId);
-        } catch (err) {
-          logger.warn("[agent-sandbox] Failed to revoke per-agent API key", {
-            agentId,
-            error: err instanceof Error ? err.message : String(err),
-          });
-        }
-      }
-
       return deletedSandbox
         ? ({ success: true, deletedSandbox } as const)
         : ({ success: false, error: "Agent not found" } as const);
     });
+
+    if (result.success) {
+      // Best-effort: revoke the per-agent API key after the row delete commits.
+      // A failure here does not un-delete the sandbox; the key just lingers as
+      // inactive data and can be cleaned by ops.
+      try {
+        await apiKeysService.revokeForAgent(agentId);
+      } catch (err) {
+        logger.warn("[agent-sandbox] Failed to revoke per-agent API key", {
+          agentId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+    }
+
+    return result;
   }
 
   /**

packages/cloud-shared/src/lib/services/memory-sandbox-provider.ts

@@ -0,0 +1,126 @@
+import { randomUUID } from "node:crypto";
+import { createServer, type Server } from "node:http";
+
+import type { SandboxCreateConfig, SandboxHandle, SandboxProvider } from "./sandbox-provider-types";
+
+interface MemorySandbox {
+  handle: SandboxHandle;
+  runtimeAgent: {
+    id: string;
+    name: string;
+    status: "active";
+  };
+  server: Server;
+}
+
+function json(body: unknown, status = 200): Response {
+  return Response.json(body, { status });
+}
+
+async function listen(server: Server): Promise<number> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("[memory-sandbox] test server did not bind to a TCP port");
+  }
+  return address.port;
+}
+
+/**
+ * Test-only sandbox provider used by cloud E2E.
+ *
+ * It exercises the real DB-backed provisioning and deletion job service without
+ * requiring Docker, SSH nodes, or live Hetzner credentials in CI. Production
+ * selection is guarded in `createSandboxProvider`.
+ */
+export class MemorySandboxProvider implements SandboxProvider {
+  private readonly sandboxes = new Map<string, MemorySandbox>();
+
+  async create(config: SandboxCreateConfig): Promise<SandboxHandle> {
+    const runtimeAgent = {
+      id: `runtime-${randomUUID()}`,
+      name: config.agentName,
+      status: "active" as const,
+    };
+
+    const server = createServer(async (req, res) => {
+      const url = new URL(req.url ?? "/", "http://127.0.0.1");
+      if (req.method === "GET" && url.pathname === "/api/health") {
+        const response = json({ success: true, status: "ok" });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/api/agents") {
+        const response = json({ success: true, agents: [runtimeAgent] });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (req.method === "POST" && url.pathname === "/api/agents") {
+        const response = json({
+          success: true,
+          data: runtimeAgent,
+        });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (
+        req.method === "POST" &&
+        url.pathname.startsWith("/api/agents/") &&
+        url.pathname.endsWith("/start")
+      ) {
+        const response = json({ success: true, data: runtimeAgent });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      const response = json({ success: false, error: "Not found" }, 404);
+      res.writeHead(response.status, Object.fromEntries(response.headers));
+      res.end(await response.text());
+    });
+
+    const port = await listen(server);
+    const sandboxId = `memory-${config.agentId}`;
+    const baseUrl = `http://127.0.0.1:${port}`;
+    const handle: SandboxHandle = {
+      sandboxId,
+      bridgeUrl: baseUrl,
+      healthUrl: `${baseUrl}/api/health`,
+      metadata: {
+        provider: "memory",
+        agentId: config.agentId,
+      },
+    };
+    this.sandboxes.set(sandboxId, { handle, runtimeAgent, server });
+    return handle;
+  }
+
+  async stop(sandboxId: string): Promise<void> {
+    const sandbox = this.sandboxes.get(sandboxId);
+    if (!sandbox) return;
+    this.sandboxes.delete(sandboxId);
+    await new Promise<void>((resolve, reject) => {
+      sandbox.server.close((error) => {
+        if (error) reject(error);
+        else resolve();
+      });
+    });
+  }
+
+  async checkHealth(handle: SandboxHandle): Promise<boolean> {
+    return this.sandboxes.has(handle.sandboxId);
+  }
+
+  async runCommand(): Promise<string> {
+    return "";
+  }
+}

packages/cloud-shared/src/lib/services/sandbox-provider.ts

@@ -17,6 +17,10 @@ export type {
  * - `DockerSandboxProvider` (SSH-into-remote-nodes) otherwise.
  */
 export async function createSandboxProvider(): Promise<SandboxProvider> {
+  if (shouldUseMemoryTestProvider()) {
+    const { MemorySandboxProvider } = await import("./memory-sandbox-provider");
+    return new MemorySandboxProvider();
+  }
   if (shouldUseLocalDockerProvider()) {
     const { LocalDockerSandboxProvider } = await import("./local-docker-sandbox-provider");
     return new LocalDockerSandboxProvider();
@@ -25,6 +29,15 @@ export async function createSandboxProvider(): Promise<SandboxProvider> {
   return new DockerSandboxProvider();
 }
 
+function shouldUseMemoryTestProvider(): boolean {
+  const env = process.env;
+  if (env.ELIZA_TEST_SANDBOX_PROVIDER !== "memory") return false;
+  if (env.NODE_ENV === "test" || env.CLOUD_E2E === "1") return true;
+  throw new Error(
+    "ELIZA_TEST_SANDBOX_PROVIDER=memory is only allowed under NODE_ENV=test or CLOUD_E2E=1",
+  );
+}
+
 function shouldUseLocalDockerProvider(): boolean {
   const env = process.env;
   if (env.MILADY_LOCAL_DOCKER_PROVIDER === "1") return true;

packages/os/usb-installer/HANDOFF.md

@@ -1,16 +1,17 @@
 # elizaOS USB Installer Handoff
 
-Last updated: 2026-05-19
+Last updated: 2026-05-20
 
 ## Current Branch
 
 - Repository: `elizaOS/eliza`
 - Worktree used for the latest proof: `/home/nubs/Git/iqlabs/elizaos-usb-prod-e2e`
-- Branch: `nubs/elizaos-live-prod-hardening-20260519`
-- PR: https://github.com/elizaOS/eliza/pull/7803
-- Verified local head: current merge head
-- Latest merged `origin/develop`: `c73f1768b68ea72b5df83efeeaadea49f812555f`
-- Latest local CI-fix validation: 2026-05-19 23:55 UTC
+- Branch: `nubs/messylinux-cloud-e2e-hardening`
+- Previous PR #7803: https://github.com/elizaOS/eliza/pull/7803 (merged)
+- Follow-up PR #7825: https://github.com/elizaOS/eliza/pull/7825
+- Verified local base: `origin/develop@27d4aee89417a4fc921c2c39ae7bd6fa3bdb9c82`
+- Latest PR head locally validated: `7497cf6c6b5e787df1ddc71ef49844cf7283c4a0`
+- Latest local CI-fix validation: 2026-05-20 05:13 UTC
 
 ## What This Package Is
 
@@ -105,6 +106,21 @@ behind the backend contract and future signed/elevated helpers.
   - OS release CI and the Linux release-packaging path now run Playwright E2E
     and run the opt-in `scsi_debug` virtual block-device proof when the runner
     kernel provides that module.
+- Additional cloud mock-stack E2E hardening added on 2026-05-20:
+  - fixed the cloud E2E repo-root resolution so the PGlite TCP bridge script
+    resolves from the repository root, not `packages/`;
+  - replaced the stale in-process control-plane mock with the real
+    `container-control-plane` sidecar and a guarded in-memory sandbox provider
+    that only activates under `NODE_ENV=test` or `CLOUD_E2E=1`;
+  - added a Node-hosted cloud-api Worker fetch adapter for the E2E harness so
+    CI exercises the generated router, real API routes, DB queue, and sidecar
+    forwarder without depending on Wrangler local runtime;
+  - fixed Node fetch forwarding for request bodies by setting `duplex: "half"`;
+  - added process-level DB pool cleanup before the fixture stops PGlite;
+  - moved best-effort per-agent API-key revocation out of the sandbox delete
+    transaction and made revocation a single delete-returning operation;
+  - updated provision/deprovision/stuck-cleanup specs to create real agents,
+    drive the real provisioning queue, and assert externally visible states.
 - Post-merge validation on 2026-05-20 after merging
   `origin/develop@c73f1768b6`:
   - `bun run verify:cloud` passed;
@@ -119,6 +135,52 @@ behind the backend contract and future signed/elevated helpers.
   - `bun run --cwd packages/os/usb-installer test:linux-virtual-usb` passed
     with `scsi_debug` cleanup verified;
   - `git diff --check` passed.
+- Final local validation on 2026-05-20 after the mock-stack E2E harness fix:
+  - `bun run --cwd packages/cloud-shared typecheck` passed;
+  - `bun run --cwd packages/cloud-api typecheck` passed;
+  - `bun run --cwd packages/cloud-api lint` passed;
+  - `bun test packages/cloud-api/webhooks/bluebubbles/route.test.ts` passed:
+    10 tests;
+  - `bun run --cwd packages/cloud-services/container-control-plane typecheck`
+    passed;
+  - `bun run --cwd packages/test/cloud-e2e typecheck` passed;
+  - `bun run --cwd packages/cloud-shared lint` passed;
+  - `bun run cloud:e2e` passed: 4 Playwright tests covering onboarding,
+    provision, deprovision, and stuck cleanup against PGlite, cloud-api,
+    cloud-frontend, the real control-plane sidecar, and the guarded memory
+    sandbox provider;
+  - `bun run --cwd packages/cloud-api test -- --runInBand` passed: 44 tests;
+  - `bun run --cwd packages/os/usb-installer typecheck` passed;
+  - `bun run --cwd packages/os/usb-installer test` passed: 9 files, 80 tests,
+    with the opt-in virtual block-device test skipped by default;
+  - `bun run --cwd packages/os/usb-installer lint` passed;
+  - `bun run --cwd packages/os/usb-installer build` passed;
+  - `bun run --cwd packages/os/usb-installer test:e2e` passed: 6 Playwright
+    tests;
+  - `bun run --cwd packages/os/usb-installer test:linux-virtual-usb` passed
+    against `scsi_debug`;
+  - `git diff --check` passed.
+- Follow-up local validation on 2026-05-20 after rebasing PR #7825 onto
+  `origin/develop@27d4aee894`:
+  - `bun run --cwd packages/cloud-shared typecheck` passed;
+  - `bun run --cwd packages/cloud-shared lint` passed;
+  - `bun run --cwd packages/cloud-api typecheck` passed;
+  - `bun run --cwd packages/test/cloud-e2e typecheck` passed;
+  - `bun run cloud:e2e` passed: 4 Playwright tests covering onboarding,
+    provision, deprovision, and stuck cleanup against PGlite, cloud-api,
+    cloud-frontend, the real control-plane sidecar, and the guarded memory
+    sandbox provider;
+  - `bun run test:cloud` passed: 279 tests across 30 files;
+  - `bun run --cwd packages/os/usb-installer typecheck` passed;
+  - `bun run --cwd packages/os/usb-installer test` passed: 9 files, 80 tests,
+    with the opt-in virtual block-device test skipped by default;
+  - `bun run --cwd packages/os/usb-installer lint` passed;
+  - `bun run --cwd packages/os/usb-installer build` passed;
+  - `bun run --cwd packages/os/usb-installer test:e2e` passed: 6 Playwright
+    tests;
+  - `bun run --cwd packages/os/usb-installer test:linux-virtual-usb` passed
+    against `scsi_debug`;
+  - `git diff --check` passed.
 - Disk cleanup on 2026-05-19:
   - removed ignored/generated stale ISO artifacts and root `dist/`;
   - removed inactive `/tmp/eliza-pr7803` temp checkout after confirming no