[iris] Install cloudflared in controller Docker image, run via docker exec

Move cloudflared from host VM bootstrap to the controller Dockerfile.
Launch it inside the running container via `docker exec -d` over SSH,
instead of as a local subprocess on the client machine.

- Dockerfile: install cloudflared binary in controller stage
- tunnel.py: launch/stop via docker exec on the controller VM
- bootstrap.py: remove host-level cloudflared installation
- vm_lifecycle.py: pass VM handle to tunnel start/stop

https://claude.ai/code/session_01RyutK1NjJZXmmKdeWbGghb

Author: claude

lib/iris/Dockerfile

@@ -77,6 +77,11 @@ FROM deps AS controller
 
 LABEL org.opencontainers.image.description="Iris controller image"
 
+# cloudflared for Cloudflare Tunnel (public dashboard access without SSH)
+RUN curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \
+        -o /usr/local/bin/cloudflared \
+    && chmod +x /usr/local/bin/cloudflared
+
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
     CMD curl -f http://localhost:10000/health || exit 1
 

lib/iris/src/iris/cluster/controller/vm_lifecycle.py

@@ -380,7 +380,7 @@ def start_controller(
         if wait_healthy(existing_vm, port, timeout=health_check_timeout):
             address = f"http://{existing_vm.internal_address}:{port}"
             logger.info("Existing controller at %s is healthy", address)
-            tunnel_handle = _maybe_start_tunnel(config, port)
+            tunnel_handle = _maybe_start_tunnel(config, port, existing_vm)
             return address, existing_vm, tunnel_handle
         logger.info("Existing controller is unhealthy, terminating and recreating")
         existing_vm.terminate()
@@ -410,7 +410,7 @@ def start_controller(
     vm.set_metadata({labels.iris_controller_address: address})
 
     # Start tunnel if configured
-    tunnel_handle = _maybe_start_tunnel(config, port)
+    tunnel_handle = _maybe_start_tunnel(config, port, vm)
 
     logger.info("Controller started at %s", address)
     if tunnel_handle:
@@ -421,13 +421,14 @@ def start_controller(
 def _maybe_start_tunnel(
     config: config_pb2.IrisClusterConfig,
     port: int,
+    vm: StandaloneWorkerHandle,
 ) -> TunnelHandle | None:
     """Start a Cloudflare Tunnel if configured. Returns None if disabled."""
     tunnel_config = _build_tunnel_config(config)
     if tunnel_config is None:
         return None
     try:
-        return start_tunnel(tunnel_config, port)
+        return start_tunnel(tunnel_config, port, vm)
     except Exception:
         logger.warning("Failed to start Cloudflare Tunnel — controller is still accessible via SSH", exc_info=True)
         return None
@@ -462,7 +463,7 @@ def restart_controller(
     if not wait_healthy(vm, port, timeout=health_check_timeout):
         raise RuntimeError(f"Controller at {address} failed health check after restart")
 
-    tunnel_handle = _maybe_start_tunnel(config, port)
+    tunnel_handle = _maybe_start_tunnel(config, port, vm)
 
     logger.info("Controller container restarted at %s", address)
     if tunnel_handle:

lib/iris/src/iris/cluster/providers/gcp/bootstrap.py

@@ -283,16 +283,6 @@ def build_worker_bootstrap_script(
     echo "[iris-controller] [1/5] Docker already installed: $(docker --version)"
 fi
 
-# Install cloudflared for Cloudflare Tunnel support (idempotent)
-if ! command -v cloudflared &> /dev/null; then
-    echo "[iris-controller] Installing cloudflared..."
-    curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \
-        -o /usr/local/bin/cloudflared && chmod +x /usr/local/bin/cloudflared
-    echo "[iris-controller] cloudflared installed: $(cloudflared --version)"
-else
-    echo "[iris-controller] cloudflared already installed: $(cloudflared --version)"
-fi
-
 echo "[iris-controller] [2/5] Ensuring Docker daemon is running..."
 sudo systemctl start docker || true
 if sudo docker info > /dev/null 2>&1; then

lib/iris/src/iris/cluster/tunnel.py

@@ -7,27 +7,36 @@
 accessible at ``marin-<nonce>.iris-ops.dev`` without a public IP address or
 SSH port-forwarding.
 
+Architecture:
+    - Cloudflare API calls (create tunnel, DNS, get token) run **client-side**
+      (on the machine running ``iris cluster start``).
+    - ``cloudflared`` runs **inside the controller container** (installed in the
+      Docker image). The container uses ``--network=host``, so
+      ``localhost:<port>`` reaches the controller.  It is launched via
+      ``docker exec`` from the client over SSH.
+
 Lifecycle:
-    1. ``start_tunnel()`` — create tunnel, set DNS CNAME, launch ``cloudflared``.
-    2. ``stop_tunnel()``  — kill ``cloudflared``, delete DNS record, delete tunnel.
+    1. ``start_tunnel()`` — create tunnel, set DNS CNAME, launch cloudflared in container.
+    2. ``stop_tunnel()``  — kill cloudflared in container, optionally delete DNS/tunnel.
 """
 
 from __future__ import annotations
 
 import hashlib
 import logging
-import subprocess
-import time
 from dataclasses import dataclass
 
 import httpx
 
+from iris.cluster.providers.types import RemoteWorkerHandle
+from iris.time_utils import Duration
+
 logger = logging.getLogger(__name__)
 
 CLOUDFLARE_API_BASE = "https://api.cloudflare.com/client/v4"
 
-# How long to wait for cloudflared to establish the tunnel connection.
-TUNNEL_HEALTH_TIMEOUT_SECONDS = 30
+# Name of the cloudflared container on the controller VM.
+CLOUDFLARED_CONTAINER_NAME = "iris-cloudflared"
 
 
 @dataclass
@@ -66,7 +75,6 @@ class TunnelHandle:
     tunnel_token: str
     dns_record_id: str
     public_url: str
-    process: subprocess.Popen | None = None
 
 
 def _cf_headers(api_token: str) -> dict[str, str]:
@@ -120,7 +128,6 @@ def _create_tunnel(
     api_token: str,
 ) -> dict:
     """Create a new Cloudflare Tunnel. Returns the tunnel object."""
-    # Generate a tunnel secret (32 bytes, base64-encoded by Cloudflare)
     import secrets
 
     tunnel_secret = secrets.token_urlsafe(32)
@@ -273,34 +280,54 @@ def _delete_tunnel(
     logger.info("Deleted tunnel %s", tunnel_id)
 
 
-def _launch_cloudflared(tunnel_token: str) -> subprocess.Popen:
-    """Launch ``cloudflared`` as a background subprocess."""
-    cmd = [
-        "cloudflared",
-        "tunnel",
-        "--no-autoupdate",
-        "run",
-        "--token",
-        tunnel_token,
-    ]
-    logger.info("Starting cloudflared tunnel connector")
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
+CONTROLLER_CONTAINER_NAME = "iris-controller"
+
+
+def _launch_cloudflared_on_vm(vm: RemoteWorkerHandle, tunnel_token: str) -> None:
+    """Launch ``cloudflared`` inside the controller container via ``docker exec``.
+
+    cloudflared is installed in the controller Docker image.  The container
+    uses ``--network=host``, so it can reach the controller at localhost.
+    """
+    # Kill any existing cloudflared inside the container
+    vm.run_command(
+        f"sudo docker exec {CONTROLLER_CONTAINER_NAME} pkill -f 'cloudflared tunnel' || true",
+        timeout=Duration.from_seconds(10),
+    )
+
+    # Launch cloudflared in the background inside the container.
+    cmd = (
+        f"sudo docker exec -d {CONTROLLER_CONTAINER_NAME} "
+        f"cloudflared tunnel --no-autoupdate run --token {tunnel_token}"
+    )
+    result = vm.run_command(cmd, timeout=Duration.from_seconds(15))
+    if result.returncode != 0:
+        raise RuntimeError(f"Failed to start cloudflared in container: {result.stderr}")
+
+    # Verify it's running
+    check = vm.run_command(
+        f"sudo docker exec {CONTROLLER_CONTAINER_NAME} pgrep -f 'cloudflared tunnel' || true",
+        timeout=Duration.from_seconds(5),
+    )
+    if not check.stdout.strip():
+        raise RuntimeError("cloudflared failed to start inside controller container")
+
+    logger.info("cloudflared started inside controller container")
+
+
+def _stop_cloudflared_on_vm(vm: RemoteWorkerHandle) -> None:
+    """Stop cloudflared inside the controller container."""
+    vm.run_command(
+        f"sudo docker exec {CONTROLLER_CONTAINER_NAME} pkill -f 'cloudflared tunnel' || true",
+        timeout=Duration.from_seconds(10),
     )
-    # Give cloudflared a moment to start and fail fast if binary is missing
-    time.sleep(2)
-    if proc.poll() is not None:
-        stderr = proc.stderr.read().decode() if proc.stderr else ""
-        raise RuntimeError(f"cloudflared exited immediately (rc={proc.returncode}): {stderr}")
-    logger.info("cloudflared started (pid=%d)", proc.pid)
-    return proc
+    logger.info("cloudflared stopped inside controller container")
 
 
-def start_tunnel(config: TunnelConfig, controller_port: int) -> TunnelHandle:
-    """Create a Cloudflare Tunnel and DNS record, then launch cloudflared.
+def start_tunnel(config: TunnelConfig, controller_port: int, vm: RemoteWorkerHandle) -> TunnelHandle:
+    """Create a Cloudflare Tunnel and DNS record, then launch cloudflared on the VM.
 
+    Cloudflare API calls run client-side; cloudflared runs on the controller VM.
     Idempotent: reuses an existing tunnel with the same name if present.
     """
     if not config.api_token:
@@ -340,8 +367,8 @@ def start_tunnel(config: TunnelConfig, controller_port: int) -> TunnelHandle:
         # 4. Upsert DNS CNAME
         dns_record_id = _upsert_dns_record(client, config.cloudflare_zone_id, fqdn, tunnel_id, config.api_token)
 
-    # 5. Launch cloudflared
-    proc = _launch_cloudflared(tunnel_token)
+    # 5. Launch cloudflared on the controller VM
+    _launch_cloudflared_on_vm(vm, tunnel_token)
 
     public_url = config.public_url
     logger.info("Tunnel active: %s", public_url)
@@ -351,31 +378,27 @@ def start_tunnel(config: TunnelConfig, controller_port: int) -> TunnelHandle:
         tunnel_token=tunnel_token,
         dns_record_id=dns_record_id,
         public_url=public_url,
-        process=proc,
     )
 
 
 def stop_tunnel(
     handle: TunnelHandle,
     config: TunnelConfig,
+    vm: RemoteWorkerHandle | None = None,
     delete_tunnel: bool = False,
 ) -> None:
-    """Stop cloudflared and optionally clean up DNS/tunnel.
+    """Stop cloudflared on the VM and optionally clean up DNS/tunnel.
 
     By default, DNS records and the tunnel are preserved so the same URL keeps
     working across controller restarts. Pass ``delete_tunnel=True`` for full
     cleanup (e.g. on ``iris cluster stop``).
     """
-    # Kill cloudflared process
-    if handle.process and handle.process.poll() is None:
-        logger.info("Stopping cloudflared (pid=%d)", handle.process.pid)
-        handle.process.terminate()
+    # Kill cloudflared on the VM
+    if vm is not None:
         try:
-            handle.process.wait(timeout=10)
-        except subprocess.TimeoutExpired:
-            handle.process.kill()
-            handle.process.wait()
-        logger.info("cloudflared stopped")
+            _stop_cloudflared_on_vm(vm)
+        except Exception:
+            logger.warning("Failed to stop cloudflared on VM", exc_info=True)
 
     if not delete_tunnel:
         return

lib/iris/tests/cluster/test_tunnel.py

@@ -10,6 +10,7 @@
 import httpx
 import pytest
 
+from iris.cluster.providers.types import CommandResult
 from iris.cluster.tunnel import (
     TunnelConfig,
     TunnelHandle,
@@ -21,6 +22,7 @@
     start_tunnel,
     stop_tunnel,
 )
+from iris.time_utils import Duration
 
 TUNNEL_MODULE = "iris.cluster.tunnel"
 
@@ -108,8 +110,6 @@ def test_find_dns_record_returns_match():
 
 def test_upsert_dns_record_creates_new():
     client = MagicMock()
-    # First call: no existing record
-    # Second call: create succeeds
     client.request.side_effect = [
         _make_cf_response([]),  # find_dns_record returns empty
         _make_cf_response({"id": "rec-new"}),  # create returns new record
@@ -141,15 +141,34 @@ def test_configure_tunnel_ingress():
     assert ingress[1]["service"] == "http_status:404"
 
 
+# ---------------------------------------------------------------------------
+# Fake VM for testing remote cloudflared launch
+# ---------------------------------------------------------------------------
+
+
+class FakeRemoteVM:
+    """Minimal fake implementing run_command for tunnel tests."""
+
+    def __init__(self, healthy: bool = True):
+        self.commands: list[str] = []
+        self._healthy = healthy
+
+    def run_command(self, command: str, timeout: Duration | None = None, on_line=None) -> CommandResult:
+        self.commands.append(command)
+        if "pgrep" in command and self._healthy:
+            return CommandResult(returncode=0, stdout="12345", stderr="")
+        return CommandResult(returncode=0, stdout="", stderr="")
+
+
 # ---------------------------------------------------------------------------
 # start_tunnel / stop_tunnel integration (mocked)
 # ---------------------------------------------------------------------------
 
 
-@patch(f"{TUNNEL_MODULE}._launch_cloudflared")
+@patch(f"{TUNNEL_MODULE}._launch_cloudflared_on_vm")
 @patch(f"{TUNNEL_MODULE}.httpx.Client")
 def test_start_tunnel_full_flow(mock_client_cls, mock_launch):
-    """start_tunnel creates tunnel, configures ingress, sets DNS, launches cloudflared."""
+    """start_tunnel creates tunnel, configures ingress, sets DNS, launches cloudflared on VM."""
     client = MagicMock()
     mock_client_cls.return_value.__enter__ = MagicMock(return_value=client)
     mock_client_cls.return_value.__exit__ = MagicMock(return_value=False)
@@ -163,10 +182,7 @@ def test_start_tunnel_full_flow(mock_client_cls, mock_launch):
         _make_cf_response({"id": "rec-xyz"}),  # create dns record
     ]
 
-    mock_proc = MagicMock()
-    mock_proc.pid = 12345
-    mock_launch.return_value = mock_proc
-
+    vm = FakeRemoteVM()
     config = TunnelConfig(
         enabled=True,
         domain="iris-ops.dev",
@@ -176,41 +192,39 @@ def test_start_tunnel_full_flow(mock_client_cls, mock_launch):
         cluster_name="test-cluster",
     )
 
-    handle = start_tunnel(config, controller_port=10000)
+    handle = start_tunnel(config, controller_port=10000, vm=vm)
 
     assert handle.tunnel_id == "tun-abc"
     assert handle.dns_record_id == "rec-xyz"
     assert handle.public_url == config.public_url
-    assert handle.process is mock_proc
-    mock_launch.assert_called_once()
+    mock_launch.assert_called_once_with(vm, "tunnel-token-string")
 
 
 def test_start_tunnel_validates_required_fields():
+    vm = FakeRemoteVM()
     config = TunnelConfig(enabled=True, api_token="", cloudflare_account_id="", cloudflare_zone_id="")
     with pytest.raises(ValueError, match="api_token"):
-        start_tunnel(config, controller_port=10000)
+        start_tunnel(config, controller_port=10000, vm=vm)
 
     config2 = TunnelConfig(enabled=True, api_token="tok", cloudflare_account_id="", cloudflare_zone_id="")
     with pytest.raises(ValueError, match="account_id"):
-        start_tunnel(config2, controller_port=10000)
+        start_tunnel(config2, controller_port=10000, vm=vm)
 
 
-def test_stop_tunnel_terminates_process():
-    proc = MagicMock()
-    proc.poll.return_value = None  # still running
+def test_stop_tunnel_kills_cloudflared_on_vm():
+    vm = FakeRemoteVM()
     handle = TunnelHandle(
         tunnel_id="tun-1",
         tunnel_token="tok",
         dns_record_id="rec-1",
         public_url="https://test.iris-ops.dev",
-        process=proc,
     )
     config = TunnelConfig(api_token="fake", cloudflare_account_id="acct", cloudflare_zone_id="zone")
 
-    stop_tunnel(handle, config, delete_tunnel=False)
+    stop_tunnel(handle, config, vm=vm, delete_tunnel=False)
 
-    proc.terminate.assert_called_once()
-    proc.wait.assert_called_once()
+    # Should have issued a pkill command
+    assert any("pkill" in cmd for cmd in vm.commands)
 
 
 @patch(f"{TUNNEL_MODULE}.httpx.Client")
@@ -230,10 +244,9 @@ def test_stop_tunnel_with_cleanup(mock_client_cls):
         tunnel_token="tok",
         dns_record_id="rec-1",
         public_url="https://test.iris-ops.dev",
-        process=None,
     )
     config = TunnelConfig(api_token="fake", cloudflare_account_id="acct", cloudflare_zone_id="zone")
 
-    stop_tunnel(handle, config, delete_tunnel=True)
+    stop_tunnel(handle, config, vm=None, delete_tunnel=True)
 
     assert client.request.call_count == 2