fix: harden cloud E2E and elizaOS USB live path

packages/cloud-api/v1/_container-control-plane-forward.ts

@@ -51,15 +51,19 @@ async function forwardControlPlaneRequest(
   configureHeaders(headers);
 
   try {
-    const upstream = await fetch(target, {
-      body:
-        c.req.method === "GET" || c.req.method === "HEAD"
-          ? undefined
-          : c.req.raw.body,
+    const body =
+      c.req.method === "GET" || c.req.method === "HEAD"
+        ? undefined
+        : c.req.raw.body;
+    const init: RequestInit & { duplex?: "half" } = {
+      body,
       headers,
       method: c.req.method,
       redirect: "manual",
-    });
+    };
+    if (body) init.duplex = "half";
+
+    const upstream = await fetch(target, init);
 
     return new Response(upstream.body, {
       headers: upstream.headers,

packages/cloud-shared/src/lib/services/api-keys.ts

@@ -254,11 +254,10 @@ export class ApiKeysService {
 
   async revokeForAgent(agentSandboxId: string): Promise<void> {
     const name = ApiKeysService.agentApiKeyName(agentSandboxId);
-    const keys = await apiKeysRepository.findByName(name);
+    const keys = await apiKeysRepository.deleteByName(name);
     for (const key of keys) {
       await this.invalidateCache(key.key_hash);
     }
-    await apiKeysRepository.deleteByName(name);
   }
 }
 

packages/cloud-shared/src/lib/services/eliza-sandbox.ts

@@ -516,7 +516,7 @@ export class ElizaSandboxService {
   }
 
   async deleteAgent(agentId: string, orgId: string): Promise<DeleteAgentResult> {
-    return dbWrite.transaction(async (tx) => {
+    const result = await dbWrite.transaction(async (tx) => {
       await this.lockLifecycle(tx, agentId, orgId);
 
       const rec = await this.getAgentForLifecycleMutation(tx, agentId, orgId);
@@ -584,23 +584,26 @@ export class ElizaSandboxService {
       `);
       const deletedSandbox = result.rows[0];
 
-      if (deletedSandbox) {
-        // Best-effort: revoke the per-agent API key. A failure here doesn't
-        // un-delete the sandbox; the key just lingers as inactive data.
-        try {
-          await apiKeysService.revokeForAgent(agentId);
-        } catch (err) {
-          logger.warn("[agent-sandbox] Failed to revoke per-agent API key", {
-            agentId,
-            error: err instanceof Error ? err.message : String(err),
-          });
-        }
-      }
-
       return deletedSandbox
         ? ({ success: true, deletedSandbox } as const)
         : ({ success: false, error: "Agent not found" } as const);
     });
+
+    if (result.success) {
+      // Best-effort: revoke the per-agent API key after the row delete commits.
+      // A failure here does not un-delete the sandbox; the key just lingers as
+      // inactive data and can be cleaned by ops.
+      try {
+        await apiKeysService.revokeForAgent(agentId);
+      } catch (err) {
+        logger.warn("[agent-sandbox] Failed to revoke per-agent API key", {
+          agentId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+    }
+
+    return result;
   }
 
   /**

packages/cloud-shared/src/lib/services/memory-sandbox-provider.ts

@@ -0,0 +1,141 @@
+import { randomUUID } from "node:crypto";
+import { createServer, type Server } from "node:http";
+import type { Socket } from "node:net";
+import { setTimeout as delay } from "node:timers/promises";
+
+import type { SandboxCreateConfig, SandboxHandle, SandboxProvider } from "./sandbox-provider-types";
+
+interface MemorySandbox {
+  handle: SandboxHandle;
+  runtimeAgent: {
+    id: string;
+    name: string;
+    status: "active";
+  };
+  server: Server;
+  sockets: Set<Socket>;
+}
+
+function json(body: unknown, status = 200): Response {
+  return Response.json(body, { status });
+}
+
+async function listen(server: Server): Promise<number> {
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(0, "127.0.0.1", () => resolve());
+  });
+  const address = server.address();
+  if (!address || typeof address === "string") {
+    throw new Error("[memory-sandbox] test server did not bind to a TCP port");
+  }
+  return address.port;
+}
+
+/**
+ * Test-only sandbox provider used by cloud E2E.
+ *
+ * It exercises the real DB-backed provisioning and deletion job service without
+ * requiring Docker, SSH nodes, or live Hetzner credentials in CI. Production
+ * selection is guarded in `createSandboxProvider`.
+ */
+export class MemorySandboxProvider implements SandboxProvider {
+  private readonly sandboxes = new Map<string, MemorySandbox>();
+
+  async create(config: SandboxCreateConfig): Promise<SandboxHandle> {
+    const runtimeAgent = {
+      id: `runtime-${randomUUID()}`,
+      name: config.agentName,
+      status: "active" as const,
+    };
+
+    const server = createServer(async (req, res) => {
+      const url = new URL(req.url ?? "/", "http://127.0.0.1");
+      if (req.method === "GET" && url.pathname === "/api/health") {
+        const response = json({ success: true, status: "ok" });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (req.method === "GET" && url.pathname === "/api/agents") {
+        const response = json({ success: true, agents: [runtimeAgent] });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (req.method === "POST" && url.pathname === "/api/agents") {
+        const response = json({
+          success: true,
+          data: runtimeAgent,
+        });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      if (
+        req.method === "POST" &&
+        url.pathname.startsWith("/api/agents/") &&
+        url.pathname.endsWith("/start")
+      ) {
+        const response = json({ success: true, data: runtimeAgent });
+        res.writeHead(response.status, Object.fromEntries(response.headers));
+        res.end(await response.text());
+        return;
+      }
+
+      const response = json({ success: false, error: "Not found" }, 404);
+      res.writeHead(response.status, Object.fromEntries(response.headers));
+      res.end(await response.text());
+    });
+    const sockets = new Set<Socket>();
+    server.on("connection", (socket) => {
+      sockets.add(socket);
+      socket.once("close", () => {
+        sockets.delete(socket);
+      });
+    });
+
+    const port = await listen(server);
+    const sandboxId = `memory-${config.agentId}`;
+    const baseUrl = `http://127.0.0.1:${port}`;
+    const handle: SandboxHandle = {
+      sandboxId,
+      bridgeUrl: baseUrl,
+      healthUrl: `${baseUrl}/api/health`,
+      metadata: {
+        provider: "memory",
+        agentId: config.agentId,
+      },
+    };
+    this.sandboxes.set(sandboxId, { handle, runtimeAgent, server, sockets });
+    return handle;
+  }
+
+  async stop(sandboxId: string): Promise<void> {
+    const sandbox = this.sandboxes.get(sandboxId);
+    if (!sandbox) return;
+    this.sandboxes.delete(sandboxId);
+    const close = new Promise<void>((resolve, reject) => {
+      sandbox.server.close((error) => {
+        if (error) reject(error);
+        else resolve();
+      });
+    });
+    sandbox.server.closeIdleConnections?.();
+    for (const socket of sandbox.sockets) {
+      socket.destroy();
+    }
+    await Promise.race([close, delay(2_000)]);
+  }
+
+  async checkHealth(handle: SandboxHandle): Promise<boolean> {
+    return this.sandboxes.has(handle.sandboxId);
+  }
+
+  async runCommand(): Promise<string> {
+    return "";
+  }
+}

packages/cloud-shared/src/lib/services/sandbox-provider.ts

@@ -17,6 +17,10 @@ export type {
  * - `DockerSandboxProvider` (SSH-into-remote-nodes) otherwise.
  */
 export async function createSandboxProvider(): Promise<SandboxProvider> {
+  if (shouldUseMemoryTestProvider()) {
+    const { MemorySandboxProvider } = await import("./memory-sandbox-provider");
+    return new MemorySandboxProvider();
+  }
   if (shouldUseLocalDockerProvider()) {
     const { LocalDockerSandboxProvider } = await import("./local-docker-sandbox-provider");
     return new LocalDockerSandboxProvider();
@@ -25,6 +29,15 @@ export async function createSandboxProvider(): Promise<SandboxProvider> {
   return new DockerSandboxProvider();
 }
 
+function shouldUseMemoryTestProvider(): boolean {
+  const env = process.env;
+  if (env.ELIZA_TEST_SANDBOX_PROVIDER !== "memory") return false;
+  if (env.NODE_ENV === "test" || env.CLOUD_E2E === "1") return true;
+  throw new Error(
+    "ELIZA_TEST_SANDBOX_PROVIDER=memory is only allowed under NODE_ENV=test or CLOUD_E2E=1",
+  );
+}
+
 function shouldUseLocalDockerProvider(): boolean {
   const env = process.env;
   if (env.MILADY_LOCAL_DOCKER_PROVIDER === "1") return true;

packages/os/linux/variants/milady-tails/Justfile

@@ -231,11 +231,13 @@ nspawn:
 boot:
     scripts/boot-qemu.sh
 
-# Write the latest built ISO to a removable USB device with guard rails.
+# Write the latest built USB image to a removable USB device with guard rails.
 usb-write device:
     scripts/usb-write.sh "{{device}}"
 
-# Write a specific ISO to a removable USB device with guard rails.
+# Write a specific ISO/USB image to a removable USB device with guard rails.
+# ISO input is rejected by default unless a neighboring .img exists or
+# ELIZAOS_CREATE_USB_IMAGE_FROM_ISO=1 is set.
 usb-write-iso device iso:
     scripts/usb-write.sh "{{device}}" "{{iso}}"
 

packages/os/linux/variants/milady-tails/README.md

@@ -113,13 +113,14 @@ Apache-2.0 where possible, dual-licensed under both where required.
 
 ## Status: Demo Branch Versus Production
 
-**Current branch status, 2026-05-19:** this branch has produced a fresh
-local ISO artifact that passed QEMU greeter/desktop/app onboarding
-validation. A prior artifact passed guarded USB flash/readback, but the
-latest validated artifact still needs repeat USB flash/readback, real
-hardware USB boot, and real USB Persistent Storage validation before it is
-called final USB-ready. Release promotion must rebuild and validate the
-exact release commit if the branch moves after the latest tested artifact.
+**Current branch status, 2026-05-20:** this branch has produced local
+artifacts that pass QEMU greeter/desktop/app startup validation. The latest
+VM USB pass also found and fixed source gaps in voice onboarding, USB-image
+partition labeling, direct-ISO USB writes, and Persistent Storage's `sudo`
+dependency. A fresh ISO plus persistence-compatible `.img` must be rebuilt
+from current HEAD and validated before it is called final USB-ready. Release
+promotion must rebuild and validate the exact release commit if the branch
+moves after the latest tested artifact.
 See [`docs/current-status.md`](./docs/current-status.md) for the exact
 validation state.
 
@@ -151,7 +152,7 @@ just build     # full clean ISO -> out/
 just build-cool # low-CPU demo build, skips offline docs, caps Docker+squashfs to 2 CPUs
 just build-demo # fastest full demo build; skips bundled offline website/docs
 just boot      # boot the latest ISO in QEMU
-just usb-write /dev/sdX # write the latest ISO with removable-disk guards
+just usb-write /dev/sdX # write the latest USB .img with removable-disk guards
 ```
 
 Set `ELIZAOS_BUILD_CPUS=2`, `ELIZAOS_MKSQUASHFS_PROCESSORS=2`, or

packages/os/linux/variants/milady-tails/build-iso.sh

@@ -377,9 +377,9 @@ echo "=== lb build (this is the long one — ~1-2h cold, faster cached) ==="
 # .img USB image — it needs UDisks (a D-Bus daemon + GI bindings) the
 # container doesn't carry. Crucially it runs *after* the .iso is fully
 # built and renamed. So a nonzero `lb build` with the .iso present means
-# only that optional post-step failed; the .iso is the deliverable and is
-# fine for QEMU testing and isohybrid USB writes. (Generating the .img is
-# revisited if/when Phase 10 bare-metal work needs it.)
+# only that optional post-step failed; the .iso remains valid for QEMU/CD-ROM
+# testing. USB persistence requires the .img layout generated by
+# create-usb-image-from-iso or an equivalent USB-image builder.
 lb_rc=0
 lb build || lb_rc=$?
 if [ "${lb_rc}" -ne 0 ]; then

packages/os/linux/variants/milady-tails/docs/build-infrastructure.md

@@ -159,7 +159,10 @@ question for v1.0.
 optional `.img` USB image and needs UDisks (a D-Bus daemon + GI
 bindings) that the container doesn't carry. It runs *after* the `.iso`
 is fully built. `build-iso.sh` treats a `lb build` failure with the
-`.iso` present as success — the `.iso` is the deliverable and is fine
-for QEMU testing and isohybrid USB writes. Generating the `.img`
-properly (UDisks-in-container, or a separate step) is revisited if/when
-Phase 10 bare-metal work needs it.
+`.iso` present as success for VM/CD-ROM testing, but the `.iso` is not
+the final USB deliverable. Persistent Storage expects the USB-image
+layout, including the upstream-compatible internal GPT system partition
+name. Release and hardware validation must generate and write the `.img`
+artifact, either in a UDisks-capable build step or via
+`ELIZAOS_CREATE_USB_IMAGE_FROM_ISO=1 scripts/usb-write.sh ...` on a Linux
+host.