up timeouts

Author: Ryan

backend/app/api/routes/deployments.py

@@ -21,6 +21,7 @@
     DeploymentUpdate,
 )
 from app.services.deploy import run_deployment, teardown_sandbox
+from app.services.uploads import get_upload_record
 
 log = logging.getLogger(__name__)
 
@@ -38,6 +39,13 @@ async def create_deployment(
     current_user: CurrentUser,
     background_tasks: BackgroundTasks,
 ) -> Deployment:
+    if payload.upload_id:
+        record = get_upload_record(payload.upload_id)
+        if record is None:
+            raise HTTPException(status_code=422, detail="Upload not found")
+        if record.owner_user_id != current_user.id:
+            raise HTTPException(status_code=403, detail="Upload does not belong to this user")
+
     deployment = Deployment(
         user_id=current_user.id,
         name=payload.name,

backend/app/api/routes/uploads.py

@@ -1,34 +1,51 @@
-import uuid
-from pathlib import Path
 from typing import Annotated
 
-from fastapi import APIRouter, File, UploadFile
+from fastapi import APIRouter, File, HTTPException, UploadFile, status
 
+from app.api.deps import CurrentUser
 from app.schemas.deployment import UploadResponse
+from app.services.uploads import (
+    UPLOAD_MAX_BYTES,
+    archive_path_for,
+    new_upload_id,
+    write_upload_metadata,
+)
 
 router = APIRouter(prefix="/uploads", tags=["uploads"])
 
-# Local-disk staging area. Swap for S3 later.
-UPLOAD_DIR = Path("uploads")
-UPLOAD_DIR.mkdir(exist_ok=True)
-
 
 @router.post("", response_model=UploadResponse)
-async def upload_code(file: Annotated[UploadFile, File(...)]) -> UploadResponse:
+async def upload_code(
+    file: Annotated[UploadFile, File(...)],
+    current_user: CurrentUser,
+) -> UploadResponse:
     """Accept a tarball / zip of the user's project directory.
 
     The CLI bundles the current folder, POSTs it here, and uses the returned
     `upload_id` when creating a deployment.
     """
-    upload_id = uuid.uuid4().hex
-    suffix = Path(file.filename or "").suffix
-    dest = UPLOAD_DIR / f"{upload_id}{suffix}"
+    upload_id = new_upload_id()
+    dest = archive_path_for(upload_id)
 
     size = 0
     with dest.open("wb") as out:
         while chunk := await file.read(1024 * 1024):
-            out.write(chunk)
             size += len(chunk)
+            if size > UPLOAD_MAX_BYTES:
+                out.close()
+                dest.unlink(missing_ok=True)
+                raise HTTPException(
+                    status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+                    detail=f"Upload too large (max {UPLOAD_MAX_BYTES // (1024 * 1024)} MB)",
+                )
+            out.write(chunk)
+    write_upload_metadata(
+        upload_id=upload_id,
+        owner_user_id=current_user.id,
+        original_filename=file.filename or dest.name,
+        stored_filename=dest.name,
+        size=size,
+    )
 
     return UploadResponse(
         upload_id=upload_id,

backend/app/core/config.py

@@ -49,6 +49,10 @@ class Settings(BaseSettings):
     dedalus_api_key: str = ""
     anthropic_api_key: str = ""
 
+    # How long deployment sandboxes should live before Modal auto-terminates
+    # them, unless explicitly torn down earlier.
+    deployment_sandbox_ttl_hours: int = 6
+
 
 @lru_cache
 def get_settings() -> Settings:

backend/app/services/deploy.py

@@ -32,8 +32,8 @@
 import re
 import shlex
 import time
-import re
 from contextlib import contextmanager
+from pathlib import Path
 from typing import Any
 
 from app.core.config import get_settings
@@ -69,6 +69,7 @@
     Sandbox,
     SandboxError,
 )
+from app.services.uploads import get_upload_record
 
 log = logging.getLogger(__name__)
 
@@ -308,6 +309,11 @@ def _clone_into_sync(sb: Sandbox, github_url: str) -> None:
         raise
 
 
+def _extract_upload_into_repo_sync(sb: Sandbox, archive_path: Path) -> None:
+    """Seed REPO_DIR from an uploaded tarball."""
+    sb.extract_upload_archive(archive_path)
+
+
 def _repo_has(sb: Sandbox, relative_path: str) -> bool:
     res = sb.exec(
         f"test -f {shlex.quote(relative_path)}",
@@ -444,21 +450,21 @@ async def run_deployment(deployment_id: str) -> None:
         f"Starting deployment: source={github_url or upload_id} model={model}",
     )
 
-    if not github_url:
+    if not github_url and not upload_id:
         await _update_deployment(
             deployment_id,
             status=DEPLOYMENT_STATUS_FAILED,
-            error=(
-                "Upload-based deployments not yet supported. "
-                "Provide `github_url` instead."
-            ),
+            error="Deployment source missing (expected github_url or upload_id).",
         )
         await _append_log(
             deployment_id,
-            "ERROR: upload-based deployments not yet supported. Use github_url.",
+            "ERROR: deployment source missing.",
+        )
+        dlog.warning(
+            "deployment source missing (github_url=%s upload_id=%s)",
+            github_url,
+            upload_id,
         )
-        dlog.warning("upload-based deployments not supported (upload_id=%s); failing fast",
-                     upload_id)
         return
 
     sb: Sandbox | None = None
@@ -479,10 +485,25 @@ async def run_deployment(deployment_id: str) -> None:
         dlog.info("sandbox acquired: id=%s", sb.object_id)
         await _append_log(deployment_id, f"Sandbox ready: {sb.object_id}")
 
-        await _append_log(deployment_id, f"Cloning {github_url}...")
-        with _step(dlog, "clone repo"):
-            await asyncio.to_thread(_clone_into_sync, sb, github_url)
-        await _append_log(deployment_id, "Repo cloned.")
+        source_description = ""
+        if github_url:
+            await _append_log(deployment_id, f"Cloning {github_url}...")
+            with _step(dlog, "clone repo"):
+                await asyncio.to_thread(_clone_into_sync, sb, github_url)
+            await _append_log(deployment_id, "Repo cloned.")
+            source_description = f"GitHub repo: {github_url}"
+        else:
+            record = get_upload_record(upload_id or "")
+            if record is None:
+                raise RuntimeError(f"Upload {upload_id} is missing from upload storage")
+            await _append_log(
+                deployment_id,
+                f"Preparing uploaded source {record.upload_id} ({record.original_filename})...",
+            )
+            with _step(dlog, "extract uploaded archive", upload_id=record.upload_id):
+                await asyncio.to_thread(_extract_upload_into_repo_sync, sb, record.archive_path)
+            await _append_log(deployment_id, "Uploaded project extracted.")
+            source_description = f"Uploaded archive: {record.upload_id}"
 
         # ------------------------------------------------------------------
         # 2. Agent #1 — analyze (with heuristic fast-path)
@@ -528,7 +549,7 @@ async def run_deployment(deployment_id: str) -> None:
                 f"Heuristic uncertain; running Agent #1 (analyze) with {model}...",
             )
             analyze_user = render_analyze_user(
-                source_description=f"GitHub repo: {github_url}",
+                source_description=source_description,
                 name=name,
                 user_env_keys=env_keys,
             )

backend/app/services/sandbox.py

@@ -213,12 +213,14 @@ def object_id(self) -> str:
     # ------------------------------------------------------------------
 
     @classmethod
-    def create(cls, *, timeout_s: int = 30 * 60) -> "Sandbox":
+    def create(cls, *, timeout_s: int | None = None) -> "Sandbox":
         # LLM keys are injected at runtime as env vars via a Modal Secret so
         # OpenClaw's `${ANTHROPIC_API_KEY}` substitution in the mounted
         # config.json resolves correctly. Keeping the keys out of the image
         # means key rotation doesn't trigger an image rebuild.
         settings = get_settings()
+        if timeout_s is None:
+            timeout_s = max(1, settings.deployment_sandbox_ttl_hours) * 60 * 60
         secret_vars: dict[str, str] = {}
         if settings.anthropic_api_key:
             secret_vars["ANTHROPIC_API_KEY"] = settings.anthropic_api_key
@@ -408,6 +410,30 @@ def wait_for_repo(self, *, max_wait_s: int = 120) -> None:
                  int((time.perf_counter() - t0) * 1000),
                  size.stdout.strip() or "?")
 
+    def extract_upload_archive(self, local_archive_path: str | Path) -> None:
+        """Copy a local tarball into the sandbox and extract it to REPO_DIR."""
+        local_path = Path(local_archive_path)
+        if not local_path.exists():
+            raise SandboxError(f"upload archive does not exist: {local_path}")
+
+        remote_path = "/tmp/dploy-upload.tar.gz"
+        log.info("copying upload archive into sandbox: %s -> %s", local_path, remote_path)
+        try:
+            self._sb.filesystem.copy_from_local(str(local_path), remote_path)
+        except Exception as e:
+            raise SandboxError(f"failed copying upload archive to sandbox: {e}") from e
+
+        self.check_exec(
+            f"rm -rf {shlex.quote(REPO_DIR)} && "
+            f"mkdir -p {shlex.quote(REPO_DIR)} && "
+            f"tar -xzf {shlex.quote(remote_path)} -C {shlex.quote(REPO_DIR)} "
+            "--no-same-owner --no-same-permissions && "
+            f"rm -f {shlex.quote(remote_path)}",
+            timeout_s=120,
+            stdout_limit=200_000,
+            stderr_limit=200_000,
+        )
+
     # ------------------------------------------------------------------
     # OpenClaw chat
     # ------------------------------------------------------------------

backend/app/services/sandbox_pool.py

@@ -136,7 +136,7 @@ def _provision_warm_sb(model: str) -> Sandbox:
     If the warmup chat fails we still return the sandbox — it's usable,
     just slower on first real use. We log a warning so it shows up.
     """
-    sb = Sandbox.create(timeout_s=30 * 60)
+    sb = Sandbox.create()
     try:
         sb.set_model(model)
         sb.start_gateway()

backend/app/services/uploads.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+UPLOAD_DIR = Path("uploads")
+UPLOAD_DIR.mkdir(exist_ok=True)
+
+UPLOAD_MAX_BYTES = 200 * 1024 * 1024  # 200 MB hard cap
+
+
+@dataclass(frozen=True)
+class UploadRecord:
+    upload_id: str
+    owner_user_id: str
+    original_filename: str
+    stored_filename: str
+    archive_path: Path
+    metadata_path: Path
+    size: int
+    created_at: str
+
+
+def new_upload_id() -> str:
+    return uuid.uuid4().hex
+
+
+def archive_path_for(upload_id: str) -> Path:
+    # Stable extension keeps extraction logic simple and avoids suffix guessing.
+    return UPLOAD_DIR / f"{upload_id}.tar.gz"
+
+
+def metadata_path_for(upload_id: str) -> Path:
+    return UPLOAD_DIR / f"{upload_id}.meta.json"
+
+
+def write_upload_metadata(
+    *,
+    upload_id: str,
+    owner_user_id: str,
+    original_filename: str,
+    stored_filename: str,
+    size: int,
+) -> Path:
+    meta = {
+        "upload_id": upload_id,
+        "owner_user_id": owner_user_id,
+        "original_filename": original_filename,
+        "stored_filename": stored_filename,
+        "size": size,
+        "created_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+    }
+    meta_path = metadata_path_for(upload_id)
+    meta_path.write_text(json.dumps(meta), encoding="utf-8")
+    return meta_path
+
+
+def get_upload_record(upload_id: str) -> UploadRecord | None:
+    archive_path = archive_path_for(upload_id)
+    meta_path = metadata_path_for(upload_id)
+    if not archive_path.exists() or not meta_path.exists():
+        return None
+    try:
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    owner_user_id = str(meta.get("owner_user_id") or "").strip()
+    if not owner_user_id:
+        return None
+    original_filename = str(meta.get("original_filename") or archive_path.name)
+    stored_filename = str(meta.get("stored_filename") or archive_path.name)
+    size = int(meta.get("size") or archive_path.stat().st_size)
+    created_at = str(meta.get("created_at") or "")
+    return UploadRecord(
+        upload_id=upload_id,
+        owner_user_id=owner_user_id,
+        original_filename=original_filename,
+        stored_filename=stored_filename,
+        archive_path=archive_path,
+        metadata_path=meta_path,
+        size=size,
+        created_at=created_at,
+    )