test(e2e): add Hetzner install suite + harden work-dir permissions

Provision ephemeral Hetzner VMs via curl-based API client (EphemeralCluster
RAII guard), run `coolify init apply`, then assert wg0/podman/firewall/coold
state over SSH. Suite is fully `#[ignore]`; env loaded from e2e-tests/.env.

Also tighten file permissions created by builder and coold:
- work_root and per-request dirs created with mode 0o700
- events.ndjson and request.json written with mode 0o600

Author: andrasbacsai

.dockerignore

@@ -0,0 +1,7 @@
+target
+Cargo.lock.bak
+.git
+
+# Never ship local secrets into an image.
+.env
+!.env.example

.gitignore

@@ -1,2 +1,6 @@
 /target
 Cargo.lock.bak
+
+# e2e-tests may keep a local .env with Hetzner tokens etc.
+.env
+!.env.example

builder/src/main.rs

@@ -26,6 +26,7 @@
 
 use std::fs::{File, OpenOptions};
 use std::io::Write;
+use std::os::unix::fs::OpenOptionsExt;
 use std::path::{Path, PathBuf};
 use std::process::ExitCode;
 
@@ -66,7 +67,14 @@ impl<'a> ProgressSink for DualSink<'a> {
 
 fn write_json_atomic(path: &Path, bytes: &[u8]) {
     let tmp = path.with_extension("json.tmp");
-    if std::fs::write(&tmp, bytes).is_ok() {
+    let ok = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .mode(0o600)
+        .open(&tmp)
+        .and_then(|mut f| f.write_all(bytes).and_then(|_| f.sync_all()));
+    if ok.is_ok() {
         let _ = std::fs::rename(tmp, path);
     }
 }
@@ -103,6 +111,7 @@ async fn main() -> ExitCode {
     let mut events = match OpenOptions::new()
         .create(true)
         .append(true)
+        .mode(0o600)
         .open(work_dir.join("events.ndjson"))
     {
         Ok(f) => f,

coold/src/builder/mod.rs

@@ -17,12 +17,13 @@
 //! children down in the same sweep.
 
 use std::collections::HashMap;
+use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
 use std::path::PathBuf;
 use std::process::Stdio;
 use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};
-use tokio::io::{AsyncBufReadExt, BufReader};
+use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
 use tokio::process::Command;
 use tokio::sync::{mpsc, Mutex, Semaphore};
 use tracing::{info, warn};
@@ -79,7 +80,12 @@ impl BuilderCtx {
     }
 
     pub async fn ensure_work_root(&self) -> std::io::Result<()> {
-        tokio::fs::create_dir_all(&self.work_root).await
+        tokio::fs::create_dir_all(&self.work_root).await?;
+        tokio::fs::set_permissions(
+            &self.work_root,
+            std::fs::Permissions::from_mode(0o700),
+        )
+        .await
     }
 
     /// Resume or reap builder transient units left by a prior coold run.
@@ -235,14 +241,29 @@ impl BuilderCtx {
 
     async fn run_build(&self, request_id: &str, req: BuildRequest) -> Result<BuildResult, BuildError> {
         let work_dir = self.work_root.join(request_id);
-        tokio::fs::create_dir_all(&work_dir)
-            .await
-            .map_err(|e| build_err(500, "setup", format!("mkdir work: {e}")))?;
+        let wd = work_dir.clone();
+        tokio::task::spawn_blocking(move || {
+            std::fs::DirBuilder::new()
+                .recursive(true)
+                .mode(0o700)
+                .create(&wd)
+        })
+        .await
+        .map_err(|e| build_err(500, "setup", format!("mkdir join: {e}")))?
+        .map_err(|e| build_err(500, "setup", format!("mkdir work: {e}")))?;
 
         let req_path = work_dir.join("request.json");
         let req_json = serde_json::to_vec(&SubprocessRequest::from_proto(&req))
             .map_err(|e| build_err(500, "setup", format!("encode request: {e}")))?;
-        tokio::fs::write(&req_path, &req_json)
+        let mut f = tokio::fs::OpenOptions::new()
+            .create(true)
+            .write(true)
+            .truncate(true)
+            .mode(0o600)
+            .open(&req_path)
+            .await
+            .map_err(|e| build_err(500, "setup", format!("open request.json: {e}")))?;
+        f.write_all(&req_json)
             .await
             .map_err(|e| build_err(500, "setup", format!("write request.json: {e}")))?;
 

e2e-tests/.env.example

@@ -0,0 +1,22 @@
+# Copy to .env and fill in. Never committed — root .gitignore covers it.
+# Values only apply when not already exported in the shell.
+
+# ─── Install suite (Hetzner-provisioned) ─────────────────────────────────────
+HETZNER_TOKEN=
+HETZNER_PROJECT=my-coold-e2e
+SSH_KEY=/Users/you/.ssh/id_ed25519
+COOLIFY_BIN=coolify
+# HETZNER_LOCATION=nbg1
+# HETZNER_IMAGE=ubuntu-24.04
+# HETZNER_SERVER_TYPE=cx23
+
+# ─── Builder suite (pre-existing cluster) ────────────────────────────────────
+# BUILDER_HOST=
+# COOLD_ONLY_HOST=
+# BUILDER_MGMT=
+# COOLD_ONLY_MGMT=
+# CENTRAL_HOST=
+# SSH_USER=root
+
+# ─── Sweeper opt-in ──────────────────────────────────────────────────────────
+# CONFIRM_SWEEP=1

e2e-tests/Cargo.toml

@@ -15,5 +15,9 @@ serde_json = { workspace = true }
 name = "builder"
 path = "tests/builder.rs"
 
+[[test]]
+name = "install"
+path = "tests/install.rs"
+
 [lib]
 path = "src/lib.rs"

e2e-tests/E2E.md

@@ -0,0 +1,104 @@
+# E2E Tests — Run Book
+
+All tests are `#[ignore]` and require live infrastructure. Always pass `--ignored --nocapture --test-threads=1`.
+
+## `.env` support
+
+Harness auto-loads `e2e-tests/.env` if present (shell-exported vars still win). Copy the template:
+
+```bash
+cp e2e-tests/.env.example e2e-tests/.env
+$EDITOR e2e-tests/.env
+```
+
+`.env` is gitignored + dockerignored.
+
+## Compile only
+
+```bash
+cargo test -p e2e-tests --no-run
+```
+
+## Suite 1 — `builder.rs` (pre-installed cluster)
+
+Requires an already-bootstrapped cluster. Env vars:
+
+```bash
+export BUILDER_HOST=<ssh-addr-of-builder-host>
+export COOLD_ONLY_HOST=<ssh-addr-of-coold-only-host>
+export BUILDER_MGMT=<wg0-ip-of-builder-host>
+export COOLD_ONLY_MGMT=<wg0-ip-of-coold-only-host>
+export CENTRAL_HOST=<ssh-addr-of-central>
+export SSH_KEY=~/.ssh/<key>
+# optional:
+export SSH_USER=root
+```
+
+### Run all in suite
+
+```bash
+cargo test -p e2e-tests --test builder -- --ignored --nocapture --test-threads=1
+```
+
+### Individual tests
+
+```bash
+cargo test -p e2e-tests --test builder pin_to_builder_host             -- --ignored --nocapture
+cargo test -p e2e-tests --test builder pin_to_coold_only_host_returns_503 -- --ignored --nocapture
+cargo test -p e2e-tests --test builder unknown_host_id_returns_503     -- --ignored --nocapture
+cargo test -p e2e-tests --test builder load_balance_picks_builder_host -- --ignored --nocapture
+cargo test -p e2e-tests --test builder build_cancel_emits_stage_cancel -- --ignored --nocapture
+cargo test -p e2e-tests --test builder coold_restart_adopts_in_flight_build -- --ignored --nocapture
+```
+
+## Suite 2 — `install.rs` (Hetzner-provisioned)
+
+Provisions VMs via Hetzner Cloud API, runs `coolify init apply`, asserts networking, destroys VMs on drop. Env vars:
+
+```bash
+export HETZNER_TOKEN=<project-scoped-token>
+export HETZNER_PROJECT=<label-value-for-cleanup-filter>
+export SSH_KEY=~/.ssh/<key>              # privkey path; .pub derived or read
+export COOLIFY_BIN=$(which coolify)      # local Go CLI path (default "coolify")
+# optional:
+export HETZNER_LOCATION=nbg1
+export HETZNER_IMAGE=ubuntu-24.04
+export HETZNER_SERVER_TYPE=cx23
+```
+
+### Both in parallel (each test provisions its own VMs)
+
+Name filter `install_` selects install_single_host + install_two_hosts and skips `cleanup_leaked_hetzner` (which would otherwise race the sweeper against live VMs).
+
+```bash
+cargo test -p e2e-tests --test install install_ -- --ignored --nocapture
+```
+
+### Single-host only
+
+```bash
+cargo test -p e2e-tests --test install install_single_host -- --ignored --nocapture
+```
+
+### Two-host only
+
+```bash
+cargo test -p e2e-tests --test install install_two_hosts -- --ignored --nocapture
+```
+
+### Leaked-resource sweeper
+
+Deletes every Hetzner server + ssh_key labeled `coolify-e2e=1` in the project. Use after a ctrl-c / crash left VMs behind. Extra `CONFIRM_SWEEP=1` gate — without it the test is a no-op, so it's safe to leave in the suite.
+
+```bash
+CONFIRM_SWEEP=1 cargo test -p e2e-tests --test install cleanup_leaked_hetzner -- --ignored --nocapture
+```
+
+## Manual spot-checks (post-install failure triage)
+
+```bash
+ssh root@<ip> wg show wg0
+ssh root@<ip> iptables -S COOLIFY-INTRA
+ssh root@<ip> nft list table bridge coolify_bridge
+ssh root@<ipA> podman exec e2e-a ping -c1 <ipB>
+```