fix(sandbox): bound ACP sandbox lifetime + self-heal vanished attach

packages/fastapi/app/config.py

@@ -53,6 +53,24 @@ class Settings(BaseSettings):
     # of session.ready_event must not hang forever. On timeout the session
     # is marked errored with an actionable message so the next send recreates.
     acp_provision_timeout_seconds: int = 180
+    # Daytona auto-deletes a sandbox once it has been continuously stopped for
+    # this many minutes — the same "continuously stopped" clock that drives
+    # auto-archive (default 7 days), so it spans the archived period too. This
+    # is the source-level bound that stops abandoned ACP boxes from
+    # accumulating (a scratch box leaked by a missed teardown after a gateway
+    # restart, or a workspace nobody returns to) and dragging the control plane
+    # down: archived boxes take ~3 minutes to wake and once enough pile up the
+    # whole account slows to a crawl. The clock only ticks while a box is
+    # stopped, so a live or recently-resumed session never trips it. Scratch
+    # (session-owned) boxes hold nothing durable → reclaimed within a day
+    # (before they even archive); persistent workspace boxes hold the user's
+    # files → a long grace period. A vanished workspace box self-heals on the
+    # next provision (a fresh one is created + relinked). MUST be positive:
+    # Daytona reads 0 as "delete immediately on stop" (NOT disabled), so any
+    # value <= 0 is clamped to "disabled" before reaching the SDK. Tunable via
+    # env without a redeploy.
+    acp_scratch_sandbox_auto_delete_minutes: int = 1440  # 1 day
+    acp_persistent_sandbox_auto_delete_minutes: int = 20160  # 14 days
     # Encrypts per-user agent credentials (AES-256-GCM; key derived via
     # SHA-256). Required for users to connect their own agent accounts —
     # there is no server-level credential fallback.

packages/fastapi/app/services/agents/daytona_runtime.py

@@ -43,6 +43,27 @@
 RUNTIME_MARKER = "/opt/harness/.acp-runtime-v1"
 
 
+def _auto_delete_minutes(persist: bool) -> int:
+    """Minutes a stopped/archived box lingers before Daytona auto-deletes it.
+
+    Persistent workspace boxes hold the user's files → a long grace period;
+    session-owned scratch boxes hold nothing durable → reclaimed quickly. This
+    is the source-level bound that keeps abandoned/leaked ACP sandboxes (a
+    scratch box leaked by a missed teardown, a workspace nobody returns to)
+    from piling up as archived boxes and dragging the control plane down.
+
+    Clamps any non-positive config to -1 (disabled): Daytona reads 0 as "delete
+    immediately on stop", which would nuke a workspace box the moment it idles —
+    so a mis-set 0 must mean "off", not "instant data loss".
+    """
+    minutes = (
+        settings.acp_persistent_sandbox_auto_delete_minutes
+        if persist
+        else settings.acp_scratch_sandbox_auto_delete_minutes
+    )
+    return minutes if minutes > 0 else -1
+
+
 def _shim_remote_path(agent: AgentDefinition) -> str:
     # Per-agent filename so pkill targets only this agent's shim and
     # multiple agents can coexist in one attached sandbox.
@@ -321,6 +342,11 @@ def provision_agent_sandbox(
                 **({"harness_persistent": "1"} if persist else {}),
             },
             auto_stop_interval=30,
+            # Reclaim the box if it sits stopped/archived past its grace
+            # period — leaked scratch boxes and abandoned workspaces otherwise
+            # accumulate as archived boxes forever. A vanished workspace box
+            # self-heals on the next provision (see _provision_once).
+            auto_delete_interval=_auto_delete_minutes(persist),
             ephemeral=False,
         )
         logger.info(

packages/fastapi/app/services/agents/session_manager.py

@@ -1549,15 +1549,39 @@ async def _provision_once(self, session: AgentSession, creds, user_ctx) -> None:
         if not adopted:
             if attach_sandbox_id is None:
                 await self._check_sandbox_cap(session.user_id)
-            session.runtime = await asyncio.to_thread(
-                provision_agent_sandbox,
-                session.user_id,
-                agent,
-                creds,
-                attach_sandbox_id,
-                None,
-                create_persistent,
-            )
+            try:
+                session.runtime = await asyncio.to_thread(
+                    provision_agent_sandbox,
+                    session.user_id,
+                    agent,
+                    creds,
+                    attach_sandbox_id,
+                    None,
+                    create_persistent,
+                )
+            except DaytonaNotFoundError:
+                # The box we tried to attach is gone from Daytona — auto-delete
+                # reclaimed an abandoned one, or it was removed out-of-band —
+                # while Convex still linked it (verify_sandbox_owner checks the
+                # Convex record, not Daytona). Heal instead of dead-ending: drop
+                # the stale link, and for a workspace-unification attach create
+                # a fresh persistent box to relink. An explicit, user-chosen
+                # harness sandbox can't be fabricated, so that re-raises.
+                if attach_sandbox_id is None or not await (
+                    self._recover_from_missing_attach(session, attach_sandbox_id)
+                ):
+                    raise
+                attach_sandbox_id, create_persistent = None, True
+                await self._check_sandbox_cap(session.user_id)
+                session.runtime = await asyncio.to_thread(
+                    provision_agent_sandbox,
+                    session.user_id,
+                    agent,
+                    creds,
+                    None,
+                    None,
+                    True,
+                )
             conn = AcpConnection(
                 session.runtime.base_url, session.runtime.headers,
             )
@@ -1700,6 +1724,45 @@ async def _resolve_sandbox_plan(
         # link it back so it becomes the workspace's unified sandbox.
         return None, True
 
+    async def _recover_from_missing_attach(
+        self, session: AgentSession, attach_sandbox_id: str,
+    ) -> bool:
+        """A box we tried to attach is gone from Daytona while Convex still
+        linked it (auto-deleted after its grace period, or removed out-of-band).
+
+        Drop the dead link so it stops being attached and stops counting
+        against the per-user cap. Return True if the caller should recover by
+        creating a fresh persistent box — a workspace-unification attach, which
+        is transparently re-created and re-linked. Return False for an explicit,
+        user-chosen harness sandbox: fabricating a replacement would silently
+        discard the box the user pointed the harness at, so surface the loss.
+        """
+        explicit = bool(
+            session.harness.sandbox_enabled
+            and session.harness.sandbox_id == attach_sandbox_id
+        )
+        await self._unlink_dead_sandbox(attach_sandbox_id)
+        logger.warning(
+            "Attached sandbox '%s' is gone (%s) — %s",
+            attach_sandbox_id,
+            "explicit harness sandbox" if explicit else "workspace unified box",
+            "surfacing the loss" if explicit else "creating a fresh one",
+        )
+        return not explicit
+
+    async def _unlink_dead_sandbox(self, daytona_id: str) -> None:
+        """Best-effort: drop the Convex sandbox row + workspace/harness links
+        for a Daytona box that no longer exists. The caller recovers regardless
+        (a fresh box is created), so a Convex hiccup here must not abort it."""
+        from app.services.convex import ConvexMutationError, run_convex_mutation
+
+        with contextlib.suppress(ConvexMutationError):
+            await run_convex_mutation(
+                self._http_client(),
+                "sandboxes:removeByDaytonaId",
+                {"daytonaSandboxId": daytona_id},
+            )
+
     async def _register_workspace_sandbox(
         self, session: AgentSession, agent,
     ) -> None:

packages/fastapi/app/services/daytona_service.py

@@ -172,10 +172,17 @@ def create_sandbox(
             **(labels or {}),
         }
 
+        # User-visible workspace/code-exec boxes hold the user's files, so give
+        # them the long grace period before Daytona reclaims an untouched one —
+        # the bound that stops stopped/archived boxes from accumulating without
+        # surprise-deleting an actively-used sandbox. Clamp non-positive to -1
+        # (disabled): Daytona reads 0 as "delete immediately on stop".
+        auto_delete = settings.acp_persistent_sandbox_auto_delete_minutes
         params = CreateSandboxFromSnapshotParams(
             snapshot=snapshot,
             language=language,
             auto_stop_interval=15,
+            auto_delete_interval=auto_delete if auto_delete > 0 else -1,
             labels=sandbox_labels,
             ephemeral=False,
         )

packages/fastapi/tests/test_sandbox_lifecycle.py

@@ -0,0 +1,142 @@
+"""Persistent-sandbox lifecycle: Daytona auto-deletes abandoned ACP boxes
+(the source-level bound that stops archived boxes piling up), and a provision
+that attaches a box Daytona has since reclaimed self-heals instead of erroring.
+
+Covers daytona_runtime._auto_delete_minutes (which grace period a new box gets)
+and AgentSessionManager._recover_from_missing_attach / _unlink_dead_sandbox
+(the heal when an attached box is gone from Daytona but still linked in Convex).
+"""
+
+import app.services.convex as convex_mod
+from app.config import settings
+from app.models import HarnessConfig
+from app.services.agents.daytona_runtime import _auto_delete_minutes
+from app.services.agents.session_manager import (
+    AgentSession,
+    AgentSessionManager,
+)
+
+
+def _session(harness: HarnessConfig) -> AgentSession:
+    return AgentSession(
+        id="s1", user_id="u1", agent_id="claude-code",
+        harness=harness, conversation_id="c1",
+    )
+
+
+class TestAutoDeleteMinutes:
+    def test_persistent_gets_long_grace(self):
+        assert (
+            _auto_delete_minutes(persist=True)
+            == settings.acp_persistent_sandbox_auto_delete_minutes
+        )
+
+    def test_scratch_reclaimed_quickly(self):
+        assert (
+            _auto_delete_minutes(persist=False)
+            == settings.acp_scratch_sandbox_auto_delete_minutes
+        )
+
+    def test_scratch_strictly_shorter_than_persistent(self):
+        # A scratch box holds nothing durable, so it must be reclaimed sooner
+        # than a workspace box that holds the user's files.
+        assert _auto_delete_minutes(False) < _auto_delete_minutes(True)
+
+    def test_non_positive_clamps_to_disabled(self, monkeypatch):
+        # Daytona reads 0 as "delete immediately on stop" — a mis-set 0 must
+        # mean "off" (-1), never instant deletion of a workspace box.
+        monkeypatch.setattr(
+            settings, "acp_persistent_sandbox_auto_delete_minutes", 0,
+        )
+        monkeypatch.setattr(
+            settings, "acp_scratch_sandbox_auto_delete_minutes", -5,
+        )
+        assert _auto_delete_minutes(persist=True) == -1
+        assert _auto_delete_minutes(persist=False) == -1
+
+
+class TestRecoverFromMissingAttach:
+    async def test_workspace_box_unlinks_and_recovers(self, monkeypatch):
+        mgr = AgentSessionManager()
+        monkeypatch.setattr(mgr, "_http_client", lambda: None)
+        unlinked: list[str] = []
+
+        async def fake_unlink(daytona_id):
+            unlinked.append(daytona_id)
+
+        monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink)
+        # A workspace-unification attach: no explicit harness sandbox.
+        h = HarnessConfig(
+            model="x", name="h", agent="claude-code", workspace_id="ws1",
+        )
+        recover = await mgr._recover_from_missing_attach(_session(h), "dt-ws")
+        assert recover is True  # caller creates a fresh persistent box
+        assert unlinked == ["dt-ws"]  # stale Convex link dropped
+
+    async def test_explicit_harness_sandbox_unlinks_but_surfaces(self, monkeypatch):
+        mgr = AgentSessionManager()
+        monkeypatch.setattr(mgr, "_http_client", lambda: None)
+        unlinked: list[str] = []
+
+        async def fake_unlink(daytona_id):
+            unlinked.append(daytona_id)
+
+        monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink)
+        # The user explicitly pointed this harness at a sandbox.
+        h = HarnessConfig(
+            model="x", name="h", agent="claude-code",
+            sandbox_enabled=True, sandbox_id="dt-explicit", workspace_id="ws1",
+        )
+        recover = await mgr._recover_from_missing_attach(
+            _session(h), "dt-explicit",
+        )
+        # Can't fabricate the user's chosen box → surface (caller re-raises),
+        # but the dead link is still cleared so the next session won't re-attach.
+        assert recover is False
+        assert unlinked == ["dt-explicit"]
+
+    async def test_explicit_sandbox_id_mismatch_treated_as_workspace(
+        self, monkeypatch,
+    ):
+        # An explicit sandbox is only "explicit" for ITS id; a workspace box on
+        # the same harness (different id) still recovers transparently.
+        mgr = AgentSessionManager()
+        monkeypatch.setattr(mgr, "_http_client", lambda: None)
+
+        async def fake_unlink(_daytona_id):
+            return None
+
+        monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink)
+        h = HarnessConfig(
+            model="x", name="h", agent="claude-code",
+            sandbox_enabled=True, sandbox_id="dt-explicit", workspace_id="ws1",
+        )
+        assert await mgr._recover_from_missing_attach(_session(h), "dt-ws") is True
+
+
+class TestUnlinkDeadSandbox:
+    async def test_calls_remove_by_daytona_id(self, monkeypatch):
+        mgr = AgentSessionManager()
+        monkeypatch.setattr(mgr, "_http_client", lambda: None)
+        calls: list[tuple] = []
+
+        async def fake_mutation(_client, path, args):
+            calls.append((path, args))
+
+        monkeypatch.setattr(convex_mod, "run_convex_mutation", fake_mutation)
+        await mgr._unlink_dead_sandbox("dt-gone")
+        assert calls == [
+            ("sandboxes:removeByDaytonaId", {"daytonaSandboxId": "dt-gone"}),
+        ]
+
+    async def test_swallows_convex_error(self, monkeypatch):
+        # A Convex hiccup must not abort the recovery (a fresh box is created
+        # regardless), so the unlink swallows ConvexMutationError.
+        mgr = AgentSessionManager()
+        monkeypatch.setattr(mgr, "_http_client", lambda: None)
+
+        async def boom(_client, _path, _args):
+            raise convex_mod.ConvexMutationError("convex down")
+
+        monkeypatch.setattr(convex_mod, "run_convex_mutation", boom)
+        await mgr._unlink_dead_sandbox("dt-gone")  # must not raise