diff --git a/packages/fastapi/app/config.py b/packages/fastapi/app/config.py index 5d3d627..f839ce1 100644 --- a/packages/fastapi/app/config.py +++ b/packages/fastapi/app/config.py @@ -53,6 +53,24 @@ class Settings(BaseSettings): # of session.ready_event must not hang forever. On timeout the session # is marked errored with an actionable message so the next send recreates. acp_provision_timeout_seconds: int = 180 + # Daytona auto-deletes a sandbox once it has been continuously stopped for + # this many minutes — the same "continuously stopped" clock that drives + # auto-archive (default 7 days), so it spans the archived period too. This + # is the source-level bound that stops abandoned ACP boxes from + # accumulating (a scratch box leaked by a missed teardown after a gateway + # restart, or a workspace nobody returns to) and dragging the control plane + # down: archived boxes take ~3 minutes to wake and once enough pile up the + # whole account slows to a crawl. The clock only ticks while a box is + # stopped, so a live or recently-resumed session never trips it. Scratch + # (session-owned) boxes hold nothing durable → reclaimed within a day + # (before they even archive); persistent workspace boxes hold the user's + # files → a long grace period. A vanished workspace box self-heals on the + # next provision (a fresh one is created + relinked). MUST be positive: + # Daytona reads 0 as "delete immediately on stop" (NOT disabled), so any + # value <= 0 is clamped to "disabled" before reaching the SDK. Tunable via + # env without a redeploy. + acp_scratch_sandbox_auto_delete_minutes: int = 1440 # 1 day + acp_persistent_sandbox_auto_delete_minutes: int = 20160 # 14 days # Encrypts per-user agent credentials (AES-256-GCM; key derived via # SHA-256). Required for users to connect their own agent accounts — # there is no server-level credential fallback. diff --git a/packages/fastapi/app/services/agents/daytona_runtime.py b/packages/fastapi/app/services/agents/daytona_runtime.py index 60c00ed..89340b4 100644 --- a/packages/fastapi/app/services/agents/daytona_runtime.py +++ b/packages/fastapi/app/services/agents/daytona_runtime.py @@ -43,6 +43,27 @@ RUNTIME_MARKER = "/opt/harness/.acp-runtime-v1" +def _auto_delete_minutes(persist: bool) -> int: + """Minutes a stopped/archived box lingers before Daytona auto-deletes it. + + Persistent workspace boxes hold the user's files → a long grace period; + session-owned scratch boxes hold nothing durable → reclaimed quickly. This + is the source-level bound that keeps abandoned/leaked ACP sandboxes (a + scratch box leaked by a missed teardown, a workspace nobody returns to) + from piling up as archived boxes and dragging the control plane down. + + Clamps any non-positive config to -1 (disabled): Daytona reads 0 as "delete + immediately on stop", which would nuke a workspace box the moment it idles — + so a mis-set 0 must mean "off", not "instant data loss". + """ + minutes = ( + settings.acp_persistent_sandbox_auto_delete_minutes + if persist + else settings.acp_scratch_sandbox_auto_delete_minutes + ) + return minutes if minutes > 0 else -1 + + def _shim_remote_path(agent: AgentDefinition) -> str: # Per-agent filename so pkill targets only this agent's shim and # multiple agents can coexist in one attached sandbox. @@ -321,6 +342,11 @@ def provision_agent_sandbox( **({"harness_persistent": "1"} if persist else {}), }, auto_stop_interval=30, + # Reclaim the box if it sits stopped/archived past its grace + # period — leaked scratch boxes and abandoned workspaces otherwise + # accumulate as archived boxes forever. A vanished workspace box + # self-heals on the next provision (see _provision_once). + auto_delete_interval=_auto_delete_minutes(persist), ephemeral=False, ) logger.info( diff --git a/packages/fastapi/app/services/agents/session_manager.py b/packages/fastapi/app/services/agents/session_manager.py index e9e4724..8922960 100644 --- a/packages/fastapi/app/services/agents/session_manager.py +++ b/packages/fastapi/app/services/agents/session_manager.py @@ -1549,15 +1549,39 @@ async def _provision_once(self, session: AgentSession, creds, user_ctx) -> None: if not adopted: if attach_sandbox_id is None: await self._check_sandbox_cap(session.user_id) - session.runtime = await asyncio.to_thread( - provision_agent_sandbox, - session.user_id, - agent, - creds, - attach_sandbox_id, - None, - create_persistent, - ) + try: + session.runtime = await asyncio.to_thread( + provision_agent_sandbox, + session.user_id, + agent, + creds, + attach_sandbox_id, + None, + create_persistent, + ) + except DaytonaNotFoundError: + # The box we tried to attach is gone from Daytona — auto-delete + # reclaimed an abandoned one, or it was removed out-of-band — + # while Convex still linked it (verify_sandbox_owner checks the + # Convex record, not Daytona). Heal instead of dead-ending: drop + # the stale link, and for a workspace-unification attach create + # a fresh persistent box to relink. An explicit, user-chosen + # harness sandbox can't be fabricated, so that re-raises. + if attach_sandbox_id is None or not await ( + self._recover_from_missing_attach(session, attach_sandbox_id) + ): + raise + attach_sandbox_id, create_persistent = None, True + await self._check_sandbox_cap(session.user_id) + session.runtime = await asyncio.to_thread( + provision_agent_sandbox, + session.user_id, + agent, + creds, + None, + None, + True, + ) conn = AcpConnection( session.runtime.base_url, session.runtime.headers, ) @@ -1700,6 +1724,45 @@ async def _resolve_sandbox_plan( # link it back so it becomes the workspace's unified sandbox. return None, True + async def _recover_from_missing_attach( + self, session: AgentSession, attach_sandbox_id: str, + ) -> bool: + """A box we tried to attach is gone from Daytona while Convex still + linked it (auto-deleted after its grace period, or removed out-of-band). + + Drop the dead link so it stops being attached and stops counting + against the per-user cap. Return True if the caller should recover by + creating a fresh persistent box — a workspace-unification attach, which + is transparently re-created and re-linked. Return False for an explicit, + user-chosen harness sandbox: fabricating a replacement would silently + discard the box the user pointed the harness at, so surface the loss. + """ + explicit = bool( + session.harness.sandbox_enabled + and session.harness.sandbox_id == attach_sandbox_id + ) + await self._unlink_dead_sandbox(attach_sandbox_id) + logger.warning( + "Attached sandbox '%s' is gone (%s) — %s", + attach_sandbox_id, + "explicit harness sandbox" if explicit else "workspace unified box", + "surfacing the loss" if explicit else "creating a fresh one", + ) + return not explicit + + async def _unlink_dead_sandbox(self, daytona_id: str) -> None: + """Best-effort: drop the Convex sandbox row + workspace/harness links + for a Daytona box that no longer exists. The caller recovers regardless + (a fresh box is created), so a Convex hiccup here must not abort it.""" + from app.services.convex import ConvexMutationError, run_convex_mutation + + with contextlib.suppress(ConvexMutationError): + await run_convex_mutation( + self._http_client(), + "sandboxes:removeByDaytonaId", + {"daytonaSandboxId": daytona_id}, + ) + async def _register_workspace_sandbox( self, session: AgentSession, agent, ) -> None: diff --git a/packages/fastapi/app/services/daytona_service.py b/packages/fastapi/app/services/daytona_service.py index 1891e7e..e7c00c7 100644 --- a/packages/fastapi/app/services/daytona_service.py +++ b/packages/fastapi/app/services/daytona_service.py @@ -172,10 +172,17 @@ def create_sandbox( **(labels or {}), } + # User-visible workspace/code-exec boxes hold the user's files, so give + # them the long grace period before Daytona reclaims an untouched one — + # the bound that stops stopped/archived boxes from accumulating without + # surprise-deleting an actively-used sandbox. Clamp non-positive to -1 + # (disabled): Daytona reads 0 as "delete immediately on stop". + auto_delete = settings.acp_persistent_sandbox_auto_delete_minutes params = CreateSandboxFromSnapshotParams( snapshot=snapshot, language=language, auto_stop_interval=15, + auto_delete_interval=auto_delete if auto_delete > 0 else -1, labels=sandbox_labels, ephemeral=False, ) diff --git a/packages/fastapi/tests/test_sandbox_lifecycle.py b/packages/fastapi/tests/test_sandbox_lifecycle.py new file mode 100644 index 0000000..7064e34 --- /dev/null +++ b/packages/fastapi/tests/test_sandbox_lifecycle.py @@ -0,0 +1,142 @@ +"""Persistent-sandbox lifecycle: Daytona auto-deletes abandoned ACP boxes +(the source-level bound that stops archived boxes piling up), and a provision +that attaches a box Daytona has since reclaimed self-heals instead of erroring. + +Covers daytona_runtime._auto_delete_minutes (which grace period a new box gets) +and AgentSessionManager._recover_from_missing_attach / _unlink_dead_sandbox +(the heal when an attached box is gone from Daytona but still linked in Convex). +""" + +import app.services.convex as convex_mod +from app.config import settings +from app.models import HarnessConfig +from app.services.agents.daytona_runtime import _auto_delete_minutes +from app.services.agents.session_manager import ( + AgentSession, + AgentSessionManager, +) + + +def _session(harness: HarnessConfig) -> AgentSession: + return AgentSession( + id="s1", user_id="u1", agent_id="claude-code", + harness=harness, conversation_id="c1", + ) + + +class TestAutoDeleteMinutes: + def test_persistent_gets_long_grace(self): + assert ( + _auto_delete_minutes(persist=True) + == settings.acp_persistent_sandbox_auto_delete_minutes + ) + + def test_scratch_reclaimed_quickly(self): + assert ( + _auto_delete_minutes(persist=False) + == settings.acp_scratch_sandbox_auto_delete_minutes + ) + + def test_scratch_strictly_shorter_than_persistent(self): + # A scratch box holds nothing durable, so it must be reclaimed sooner + # than a workspace box that holds the user's files. + assert _auto_delete_minutes(False) < _auto_delete_minutes(True) + + def test_non_positive_clamps_to_disabled(self, monkeypatch): + # Daytona reads 0 as "delete immediately on stop" — a mis-set 0 must + # mean "off" (-1), never instant deletion of a workspace box. + monkeypatch.setattr( + settings, "acp_persistent_sandbox_auto_delete_minutes", 0, + ) + monkeypatch.setattr( + settings, "acp_scratch_sandbox_auto_delete_minutes", -5, + ) + assert _auto_delete_minutes(persist=True) == -1 + assert _auto_delete_minutes(persist=False) == -1 + + +class TestRecoverFromMissingAttach: + async def test_workspace_box_unlinks_and_recovers(self, monkeypatch): + mgr = AgentSessionManager() + monkeypatch.setattr(mgr, "_http_client", lambda: None) + unlinked: list[str] = [] + + async def fake_unlink(daytona_id): + unlinked.append(daytona_id) + + monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink) + # A workspace-unification attach: no explicit harness sandbox. + h = HarnessConfig( + model="x", name="h", agent="claude-code", workspace_id="ws1", + ) + recover = await mgr._recover_from_missing_attach(_session(h), "dt-ws") + assert recover is True # caller creates a fresh persistent box + assert unlinked == ["dt-ws"] # stale Convex link dropped + + async def test_explicit_harness_sandbox_unlinks_but_surfaces(self, monkeypatch): + mgr = AgentSessionManager() + monkeypatch.setattr(mgr, "_http_client", lambda: None) + unlinked: list[str] = [] + + async def fake_unlink(daytona_id): + unlinked.append(daytona_id) + + monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink) + # The user explicitly pointed this harness at a sandbox. + h = HarnessConfig( + model="x", name="h", agent="claude-code", + sandbox_enabled=True, sandbox_id="dt-explicit", workspace_id="ws1", + ) + recover = await mgr._recover_from_missing_attach( + _session(h), "dt-explicit", + ) + # Can't fabricate the user's chosen box → surface (caller re-raises), + # but the dead link is still cleared so the next session won't re-attach. + assert recover is False + assert unlinked == ["dt-explicit"] + + async def test_explicit_sandbox_id_mismatch_treated_as_workspace( + self, monkeypatch, + ): + # An explicit sandbox is only "explicit" for ITS id; a workspace box on + # the same harness (different id) still recovers transparently. + mgr = AgentSessionManager() + monkeypatch.setattr(mgr, "_http_client", lambda: None) + + async def fake_unlink(_daytona_id): + return None + + monkeypatch.setattr(mgr, "_unlink_dead_sandbox", fake_unlink) + h = HarnessConfig( + model="x", name="h", agent="claude-code", + sandbox_enabled=True, sandbox_id="dt-explicit", workspace_id="ws1", + ) + assert await mgr._recover_from_missing_attach(_session(h), "dt-ws") is True + + +class TestUnlinkDeadSandbox: + async def test_calls_remove_by_daytona_id(self, monkeypatch): + mgr = AgentSessionManager() + monkeypatch.setattr(mgr, "_http_client", lambda: None) + calls: list[tuple] = [] + + async def fake_mutation(_client, path, args): + calls.append((path, args)) + + monkeypatch.setattr(convex_mod, "run_convex_mutation", fake_mutation) + await mgr._unlink_dead_sandbox("dt-gone") + assert calls == [ + ("sandboxes:removeByDaytonaId", {"daytonaSandboxId": "dt-gone"}), + ] + + async def test_swallows_convex_error(self, monkeypatch): + # A Convex hiccup must not abort the recovery (a fresh box is created + # regardless), so the unlink swallows ConvexMutationError. + mgr = AgentSessionManager() + monkeypatch.setattr(mgr, "_http_client", lambda: None) + + async def boom(_client, _path, _args): + raise convex_mod.ConvexMutationError("convex down") + + monkeypatch.setattr(convex_mod, "run_convex_mutation", boom) + await mgr._unlink_dead_sandbox("dt-gone") # must not raise