Merge pull request #7746 from elizaOS/feat/agent-delete-via-job-queue

Author: lalalune

packages/cloud-api/v1/eliza/agents/[agentId]/route.ts

@@ -18,6 +18,7 @@ import { getPreferredElizaAgentWebUiUrl } from "@/lib/eliza-agent-web-ui";
 import { adminService } from "@/lib/services/admin";
 import { reusesExistingElizaCharacter } from "@/lib/services/eliza-agent-config";
 import { elizaSandboxService } from "@/lib/services/eliza-sandbox";
+import { provisioningJobService } from "@/lib/services/provisioning-jobs";
 import { getStewardAgent } from "@/lib/services/steward-client";
 import type {
   AgentAdminDetailsDto,
@@ -364,63 +365,63 @@ app.delete("/", async (c) => {
       return c.json({ success: false, error: "Agent not found" }, 404);
     }
 
-    if (existing.node_id && existing.sandbox_id) {
-      const forwarded = await deleteDockerBackedAgentViaControlPlane(
-        c,
-        user,
-        agentId,
+    if (existing.status === "provisioning") {
+      return c.json(
+        { success: false, error: "Agent provisioning is in progress" },
+        409,
       );
-      if (forwarded) return forwarded;
     }
 
-    const deleted = await elizaSandboxService.deleteAgent(
+    // Async delete via the same job-queue path agent_provision uses. This
+    // moves the SSH stop, Neon cleanup, and per-agent key revoke off the
+    // request thread so a slow / unreachable Hetzner core can no longer
+    // make the API hang or silently return 200 while the container lives
+    // on. Idempotent: a second DELETE while a job is in flight reuses
+    // the existing one.
+    const enqueueResult = await provisioningJobService.enqueueAgentDeleteOnce({
       agentId,
-      user.organization_id,
-    );
-    if (!deleted.success) {
-      const status =
-        deleted.error === "Agent not found"
-          ? 404
-          : deleted.error === "Agent provisioning is in progress"
-            ? 409
-            : 500;
-      return c.json({ success: false, error: deleted.error }, status);
-    }
-
-    const characterId = deleted.deletedSandbox.character_id;
-    const reusesExistingCharacter = reusesExistingElizaCharacter(
-      deleted.deletedSandbox.agent_config,
-    );
+      organizationId: user.organization_id,
+      userId: user.id,
+    });
 
-    if (characterId && !reusesExistingCharacter) {
-      try {
-        await userCharactersRepository.delete(characterId);
-        logger.info("[agent-api] Cleaned up linked character after delete", {
-          agentId,
-          characterId,
-        });
-      } catch (characterErr) {
-        logger.warn(
-          "[agent-api] Failed to clean up linked character after delete",
-          {
-            agentId,
-            characterId,
-            error:
-              characterErr instanceof Error
-                ? characterErr.message
-                : String(characterErr),
-          },
-        );
-      }
-    }
+    // Best-effort wake of the worker so the user does not wait for the
+    // next cron tick. Same pattern as the provision path.
+    void provisioningJobService.triggerImmediate(c.env).catch(() => {
+      // Logged inside the service; nothing actionable here.
+    });
 
-    logger.info("[agent-api] Agent deleted", {
+    logger.info("[agent-api] Agent delete enqueued", {
       agentId,
       orgId: user.organization_id,
+      jobId: enqueueResult.job.id,
+      created: enqueueResult.created,
     });
 
-    return c.json({ success: true });
+    return c.json(
+      {
+        success: true,
+        created: enqueueResult.created,
+        alreadyInProgress: !enqueueResult.created,
+        message: enqueueResult.created
+          ? "Delete job created. Poll the job endpoint for status."
+          : "Delete is already in progress.",
+        data: {
+          jobId: enqueueResult.job.id,
+          agentId,
+          status: enqueueResult.job.status,
+        },
+        polling: {
+          endpoint: `/api/v1/jobs/${enqueueResult.job.id}`,
+          intervalMs: 5_000,
+          expectedDurationMs: 30_000,
+        },
+      },
+      202,
+    );
   } catch (error) {
+    if (error instanceof Error && error.message === "Agent not found") {
+      return c.json({ success: false, error: "Agent not found" }, 404);
+    }
     logger.error("[agent-api] DELETE /agents/:agentId error", { error });
     return failureResponse(c, error);
   }

packages/cloud-shared/src/db/schemas/agent-sandboxes.ts

@@ -54,7 +54,21 @@ export type AgentSandboxStatus =
   | "running"
   | "stopped"
   | "disconnected"
-  | "error";
+  | "error"
+  /**
+   * Row is queued for async deletion. An `agent_delete` job has been
+   * enqueued in `jobs`; the provisioning worker will SSH the core, stop
+   * the container, and then DELETE the row. UI must treat this as
+   * "soon-to-be-gone" — no mutations should be accepted while in this
+   * state.
+   */
+  | "deletion_pending"
+  /**
+   * Async deletion exhausted retries (e.g. SSH unreachable for the core
+   * hosting this sandbox). The container may still be running on the
+   * core; ops must investigate. Row stays so the failure is visible.
+   */
+  | "deletion_failed";
 
 export type AgentBillingStatus = "active" | "warning" | "suspended" | "shutdown_pending" | "exempt";
 

packages/cloud-shared/src/lib/services/__tests__/docker-sandbox-already-gone.test.ts

@@ -0,0 +1,64 @@
+/**
+ * Covers the substring matcher that decides whether a failed `docker stop`
+ * / `docker rm` error indicates the container is already absent. This is
+ * the pivot of the prod fix shipped in PR #BIG — without it, both-calls-
+ * failed used to silently leave zombie containers on the cores; with it,
+ * both-calls-failed throws only when the failures are unrelated to "gone".
+ */
+import { describe, expect, test } from "bun:test";
+import { isAlreadyGoneMessage } from "../docker-error-classifier";
+
+describe("isAlreadyGoneMessage", () => {
+  test('recognizes "No such container" (Docker 24)', () => {
+    expect(
+      isAlreadyGoneMessage(
+        "Error response from daemon: No such container: agent-abc123",
+      ),
+    ).toBe(true);
+  });
+
+  test('recognizes "not found" (older Docker)', () => {
+    expect(isAlreadyGoneMessage("Container not found: agent-abc")).toBe(true);
+  });
+
+  test('recognizes "already gone"', () => {
+    expect(isAlreadyGoneMessage("container already gone before stop")).toBe(
+      true,
+    );
+  });
+
+  test('recognizes "no longer exists"', () => {
+    expect(
+      isAlreadyGoneMessage("the named container no longer exists on host"),
+    ).toBe(true);
+  });
+
+  test("case-insensitive", () => {
+    expect(isAlreadyGoneMessage("NO SUCH CONTAINER: AGENT-1")).toBe(true);
+  });
+
+  test("returns false for SSH connection failure", () => {
+    expect(
+      isAlreadyGoneMessage(
+        "ssh: connect to host 138.201.80.125 port 22: Connection timed out",
+      ),
+    ).toBe(false);
+  });
+
+  test("returns false for Docker daemon down", () => {
+    expect(
+      isAlreadyGoneMessage(
+        "Cannot connect to the Docker daemon at unix:///var/run/docker.sock",
+      ),
+    ).toBe(false);
+  });
+
+  test("returns false for permission denied", () => {
+    expect(isAlreadyGoneMessage("Permission denied (publickey)")).toBe(false);
+  });
+
+  test("returns false for empty / unrelated text", () => {
+    expect(isAlreadyGoneMessage("")).toBe(false);
+    expect(isAlreadyGoneMessage("some unrelated error")).toBe(false);
+  });
+});

packages/cloud-shared/src/lib/services/docker-error-classifier.ts

@@ -0,0 +1,24 @@
+/**
+ * Tiny dep-free helpers to classify errors returned by `docker` / SSH so
+ * the rest of the sandbox provider can stay readable. Extracted from
+ * `docker-sandbox-provider.ts` only so the helpers can be unit-tested
+ * without pulling in plugin-sql / drizzle / @elizaos/core at import time.
+ */
+
+/**
+ * Matches Docker / SSH error messages that mean "the thing we tried to
+ * stop is no longer there". Used by `DockerSandboxProvider.stop()` to
+ * treat both-calls-failed as success when the container was already gone
+ * before we got the SSH window. Substring match because docker error
+ * formatting drifts across versions ("No such container", "is not
+ * running", etc.).
+ */
+export function isAlreadyGoneMessage(message: string): boolean {
+  const normalized = message.toLowerCase();
+  return (
+    normalized.includes("no such container") ||
+    normalized.includes("not found") ||
+    normalized.includes("already gone") ||
+    normalized.includes("no longer exists")
+  );
+}