feat(builder): tighten sandbox + reap orphan transient units

Two improvements on top of the live-deploy fixes from earlier today:

1. Revert ProtectSystem=full → strict with an explicit ReadWritePaths
   allowlist. Under `full`, /etc and /var were writable to a root
   builder subprocess, which let a buildah/git CVE in a malicious repo
   persist via /etc/cron.d, sudoers, systemd drop-ins, etc. Under
   `strict`, the only writable paths are the per-build work dir,
   /var/lib/containers (buildah image store), /run/containers,
   /run/netavark, and /run/lock (netavark lock). /tmp and /var/tmp
   stay ephemeral via PrivateTmp; /home and /root via ProtectHome.

   Also adds a defense-in-depth set that doesn't require path
   enumeration: NoNewPrivileges, RestrictSUIDSGID, LockPersonality,
   RestrictRealtime, RestrictNamespaces, SystemCallArchitectures=native.
   CapabilityBoundingSet and SystemCallFilter are deferred until the
   ReadWritePaths set is confirmed stable in the field.

2. Add BuilderCtx::reap_orphan_units() called on coold startup.
   systemd-run --pipe --wait tears the transient unit down on SIGTERM
   (graceful coold stop), but SIGKILL / OOM / host crash leaves the
   unit running under PID 1 until RuntimeMaxSec hits. The sweeper
   enumerates coolify-build-*.service on startup, stops any that are
   still active, and resets failed state so subsequent dispatches can
   reuse names cleanly.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: andrasbacsai

coold/src/builder/mod.rs

@@ -72,6 +72,53 @@ impl BuilderCtx {
         tokio::fs::create_dir_all(&self.work_root).await
     }
 
+    /// Stop any `coolify-build-*.service` transient units left behind by a
+    /// prior coold run. A clean coold shutdown lets `systemd-run --pipe
+    /// --wait` tear its unit down via SIGTERM propagation, but SIGKILL / OOM
+    /// / host crash leaves the unit running under PID 1 until RuntimeMaxSec
+    /// hits. This sweeper reclaims them on startup.
+    pub async fn reap_orphan_units() {
+        let out = match Command::new("systemctl")
+            .args([
+                "list-units",
+                "--all",
+                "--no-legend",
+                "--plain",
+                "--type=service",
+                "coolify-build-*.service",
+            ])
+            .output()
+            .await
+        {
+            Ok(o) => o,
+            Err(e) => {
+                warn!(error = %e, "systemctl list-units failed; skipping orphan sweep");
+                return;
+            }
+        };
+        let stdout = String::from_utf8_lossy(&out.stdout);
+        let names: Vec<String> = stdout
+            .lines()
+            .filter_map(|l| l.split_whitespace().next().map(str::to_owned))
+            .filter(|n| n.starts_with("coolify-build-") && n.ends_with(".service"))
+            .collect();
+        if names.is_empty() {
+            return;
+        }
+        warn!(count = names.len(), units = ?names,
+              "reaping orphaned builder units from a prior coold run");
+        let _ = Command::new("systemctl")
+            .arg("stop")
+            .args(&names)
+            .status()
+            .await;
+        let _ = Command::new("systemctl")
+            .arg("reset-failed")
+            .args(&names)
+            .status()
+            .await;
+    }
+
     /// Spawn a build. Returns immediately after the subprocess is started;
     /// the reader task stays alive in a detached tokio task and sends the
     /// final `Response` frame on `tx` when the subprocess exits.
@@ -156,23 +203,45 @@ impl BuilderCtx {
             .arg(format!("CPUQuota={}", self.cpu_quota))
             .arg("-p")
             .arg("PrivateTmp=yes")
-            // ProtectSystem=full keeps /usr, /boot, /efi read-only but leaves
-            // /var, /run, /etc writable — buildah needs to touch
-            // /var/lib/containers/storage and /run/containers (netavark locks,
-            // image overlay mountpoints), so "strict" breaks the build with
-            // "open /run/lock/netavark.lock: read-only file system".
+            // ProtectSystem=strict mounts / read-only except /dev, /proc, /sys
+            // and the explicit ReadWritePaths below. Full allowlist:
+            //   * `work_dir`                — git clone + generated Containerfile
+            //   * /var/lib/containers       — buildah image store + overlays
+            //   * /run/containers           — buildah/libpod runtime state
+            //   * /run/netavark             — network-plugin state
+            //   * /run/lock                 — netavark.lock
+            // /tmp + /var/tmp are ephemeral via PrivateTmp; /home and /root
+            // are hidden via ProtectHome.
             .arg("-p")
-            .arg("ProtectSystem=full")
+            .arg("ProtectSystem=strict")
             .arg("-p")
             .arg("ProtectHome=yes")
             .arg("-p")
             .arg(format!("ReadWritePaths={}", work_dir.display()))
             .arg("-p")
-            .arg("ReadWritePaths=/var/lib/containers/storage")
+            .arg("ReadWritePaths=/var/lib/containers")
             .arg("-p")
             .arg("ReadWritePaths=/run/containers")
             .arg("-p")
+            .arg("ReadWritePaths=/run/netavark")
+            .arg("-p")
             .arg("ReadWritePaths=/run/lock")
+            // Defense-in-depth. Builder runs as root for buildah's benefit,
+            // so lock down everything not strictly required. CAP_* trim and
+            // SystemCallFilter are deferred until the ReadWritePaths set is
+            // stable (easier to iterate on one dimension at a time).
+            .arg("-p")
+            .arg("NoNewPrivileges=yes")
+            .arg("-p")
+            .arg("RestrictSUIDSGID=yes")
+            .arg("-p")
+            .arg("LockPersonality=yes")
+            .arg("-p")
+            .arg("RestrictRealtime=yes")
+            .arg("-p")
+            .arg("RestrictNamespaces=yes")
+            .arg("-p")
+            .arg("SystemCallArchitectures=native")
             .arg("--")
             .arg(&self.builder_bin)
             .arg(&req_path)

coold/src/grpc/client.rs

@@ -49,6 +49,7 @@ pub async fn run(config: Config, podman: PodmanClient) -> Result<()> {
         ctx.ensure_work_root()
             .await
             .with_context(|| format!("mkdir -p {}", config.builder_work_dir.display()))?;
+        BuilderCtx::reap_orphan_units().await;
         info!(
             work_dir = %config.builder_work_dir.display(),
             capacity = config.builder_capacity,