feat: add durable identity-validated microVM cleanup

.github/workflows/test-cloud-hypervisor.yml

@@ -252,15 +252,16 @@ jobs:
         if: always()
         run: |
           set -euo pipefail
-          while read -r namespace _; do
-            case "$namespace" in
-              awfvm-*) sudo ip netns delete "$namespace" ;;
-            esac
-          done < <(sudo ip netns list)
           if sudo ip netns list | grep -q '^awfvm-'; then
+            sudo ip netns list >&2
             echo "::error::Cloud Hypervisor namespace residue remains after cleanup"
             exit 1
           fi
+          if sudo iptables -S DOCKER-USER | grep -q -- '--comment awf:awf_vm_'; then
+            sudo iptables -S DOCKER-USER >&2
+            echo "::error::Cloud Hypervisor bridge-rule residue remains after cleanup"
+            exit 1
+          fi
           # /sys/fs/cgroup/awf-cloud-hypervisor is a parent cgroup that
           # persists across the whole job; only per-run sub-cgroups are
           # created one level inside it (see cgroupPath in

docs/cloud-hypervisor-foundation.md

@@ -95,7 +95,37 @@ AWF performs these steps for each run:
     and delete the exact per-run account.
 
 Cleanup is idempotent and aggregates errors so one cleanup failure does not
-skip later cleanup steps.
+skip later cleanup steps. Before the first privileged per-run resource is
+created, AWF atomically writes a root-owned mode-`0600` recovery record under
+`/run/awf-cloud-hypervisor/pending-cleanup/`, itself a root-owned mode-`0700`
+directory. The record contains the owning AWF PID, `/proc` start time,
+executable identity, exact resource names, and immutable inode/ifindex
+identities captured immediately after each attested artifact snapshot,
+namespace, interface, run directory, cgroup, VMM, and `virtiofsd` process
+becomes live. The host bridge-forwarding
+rule is tagged with a per-run iptables comment and recorded by its exact tuple,
+so concurrent runs do not share an anonymously owned rule. Dedicated VMM
+account and device-ACL intent is persisted before `useradd`
+or `setfacl` runs. Recovery validates the account's random name, run-specific
+passwd metadata, numeric uid/gid, and exact numeric ACL entries before removing
+them.
+Any staged virtio-fs bind mounts are recorded by mount ID, device, root, target,
+filesystem type, and source; stale recovery revalidates and unmounts them
+deepest-first before removing their inode-validated share directory.
+The backend deletes the shared verified-artifact snapshot before completing
+each successfully cleaned manager record, including failed boot attempts.
+
+Every subsequent Cloud Hypervisor startup reaps stale records before creating
+its own resources. A record whose owner still has the same PID, start time,
+executable inode, credentials, and network namespace is active and is skipped,
+so concurrent sibling runs cannot reap one another. For an abandoned record,
+AWF revalidates every existing resource and process immediately before acting.
+It never treats a name or PID alone as ownership evidence: PID reuse, a changed
+namespace/interface inode or ifindex, an uncommitted launch identity, malformed
+state, or an unsafe record mode stops cleanup, reports an error, and preserves
+the record and resources for diagnosis. The record is removed only after normal
+teardown succeeds. `--keep-containers` is an explicit diagnostic opt-out: its
+record is removed while the requested resources remain preserved.
 
 ## Security boundaries
 

scripts/ci/cloud-hypervisor-live-smoke.sh

@@ -7,8 +7,9 @@ set -euo pipefail
 # This covers allowed/blocked domains, direct
 # egress, arbitrary TCP, DNS, metadata IP, mandatory API-proxy reflect with
 # secret-sentinel absence, live workspace sharing incl. symlinks/permissions,
-# exit-code propagation, timeout, SIGTERM cancellation, partial-start
-# rollback, keep/preserve diagnostics, plus backend-specific live checks:
+# exit-code propagation, timeout, SIGTERM cancellation, abrupt process-death
+# recovery, partial-start rollback, keep/preserve diagnostics, plus
+# backend-specific live checks:
 #
 #   - device-assumptions: confirms eth0, the sole /dev/vda block disk, and
 #     virtio-fs workspace layout documented in Part 6.
@@ -100,6 +101,11 @@ assert_no_residue() {
     echo "Cloud Hypervisor veth/TAP residue detected" >&2
     return 1
   fi
+  if sudo iptables -S DOCKER-USER | grep -q -- '--comment awf:awf_vm_'; then
+    sudo iptables -S DOCKER-USER >&2
+    echo "Cloud Hypervisor per-run bridge rule residue detected" >&2
+    return 1
+  fi
   # $CGROUP_ROOT (.../awf-cloud-hypervisor) is a *parent* cgroup that
   # persists across runs; only per-run sub-cgroups live one level
   # inside it (see cgroupPath in src/cloud-hypervisor/manager.ts). Any
@@ -533,6 +539,65 @@ if [ "$cleanup_ms" -gt "$CLEANUP_CEILING_MS" ]; then
   exit 1
 fi
 
+# SIGKILL cannot run in-process finally/signal cleanup. Kill the root AWF Node
+# process itself, prove its VMM/netns/cgroup survive, then let the next ordinary
+# invocation recover them through the durable identity-validated registry.
+crash_work="$RUN_ROOT/process-death/work"
+crash_workspace="$RUN_ROOT/process-death/workspace"
+crash_audit="$RUN_ROOT/process-death/audit"
+mkdir -p "$crash_work" "$crash_workspace" "$crash_audit"
+(
+  export GITHUB_WORKSPACE="$crash_workspace"
+  export OPENAI_API_KEY="$SECRET_SENTINEL"
+  exec sudo -E node "$ROOT/dist/cli.js" \
+    "${COMMON[@]}" \
+    --work-dir "$crash_work" \
+    --audit-dir "$crash_audit" \
+    -- 'sleep 300'
+) >"$RUN_ROOT/process-death/stdout.log" 2>"$RUN_ROOT/process-death/stderr.log" &
+crash_wrapper_pid=$!
+crash_node_pid=
+for _ in $(seq 1 90); do
+  for candidate in $(pgrep -f "node $ROOT/dist/cli.js.*--work-dir $crash_work" || true); do
+    [ "$candidate" = "$crash_wrapper_pid" ] && continue
+    candidate_exe=$(sudo readlink "/proc/$candidate/exe" 2>/dev/null || true)
+    case "$candidate_exe" in
+      */node) crash_node_pid=$candidate; break ;;
+    esac
+  done
+  if [ -n "$crash_node_pid" ] &&
+     sudo ip netns list | grep -q '^awfvm-' &&
+     sudo find /run/awf-cloud-hypervisor/pending-cleanup -maxdepth 1 -name '*.json' | grep -q . &&
+     sudo find "$CGROUP_ROOT" -mindepth 1 -maxdepth 1 -type d | grep -q . &&
+     pgrep -f "$ARTIFACT_DIR/cloud-hypervisor --api-socket" >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+[ -n "$crash_node_pid" ] || {
+  echo "process-death: AWF process did not become live" >&2
+  exit 1
+}
+sudo kill -KILL "$crash_node_pid"
+set +e
+wait "$crash_wrapper_pid"
+crash_status=$?
+set -e
+[ "$crash_status" -ne 0 ] || {
+  echo "process-death: SIGKILL unexpectedly returned success" >&2
+  exit 1
+}
+sudo ip netns list | grep -q '^awfvm-' || {
+  echo "process-death: abrupt exit did not leave the expected recovery fixture" >&2
+  exit 1
+}
+pgrep -f "$ARTIFACT_DIR/cloud-hypervisor --api-socket" >/dev/null || {
+  echo "process-death: VMM did not survive abrupt owner death" >&2
+  exit 1
+}
+run_case process-death-reaper 0 'true'
+assert_no_residue
+
 keep_work="$RUN_ROOT/keep/work"
 keep_workspace="$RUN_ROOT/keep/workspace"
 keep_audit="$RUN_ROOT/keep/audit"
@@ -586,6 +651,16 @@ sudo find "$keep_audit/cloud-hypervisor" -type f -size +1048576c -print -quit \
     exit 1
   }
 
+readarray -t keep_forward_rule < <(
+  sudo node -e '
+    const plan = require(process.argv[1]);
+    console.log(plan.infrastructureBridge);
+    console.log(plan.hostForwardRuleComment);
+  ' "$keep_audit/cloud-hypervisor/network-plan.json"
+)
+sudo iptables -t filter -D DOCKER-USER \
+  -i "${keep_forward_rule[0]}" -o "${keep_forward_rule[0]}" \
+  -m comment --comment "${keep_forward_rule[1]}" -j ACCEPT
 while read -r namespace _; do
   case "$namespace" in
     awfvm-*) sudo ip netns delete "$namespace" ;;

src/cloud-hypervisor-runtime-backend.test.ts

@@ -152,6 +152,7 @@ function harness(overrides: Partial<CloudHypervisorRuntimeBackendDependencies> =
     endStdin: jest.fn().mockResolvedValue(undefined),
     collectDiagnostics: jest.fn().mockResolvedValue(undefined),
     collectGuestOutputAudit: jest.fn().mockResolvedValue(undefined),
+    completeCleanupRecord: jest.fn().mockResolvedValue(undefined),
     stop: jest.fn(async (options?: { beforeCleanup?: () => Promise<void> }) => {
       order.push('vm-stop');
       await options?.beforeCleanup?.();
@@ -398,6 +399,30 @@ describe('Cloud Hypervisor runtime backend', () => {
     expect(manager.collectGuestOutputAudit).not.toHaveBeenCalled();
   });
 
+  it('reuses the CLI preflight snapshot when workflow startup begins', async () => {
+    const { deps, manager } = harness();
+    const backend = createBackend(config(), deps);
+
+    await backend.preflight();
+    await backend.start('/tmp/awf', ['github.com']);
+    await backend.stop();
+
+    expect(deps.preflight).toHaveBeenCalledTimes(1);
+    expect(deps.createManager).toHaveBeenCalledWith(
+      expect.anything(),
+      '/tmp/awf',
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      undefined,
+      preflightResult,
+    );
+    expect(deps.removeArtifactSnapshot).toHaveBeenCalledWith('/snapshot');
+    expect(manager.completeCleanupRecord).toHaveBeenCalledTimes(1);
+    expect((deps.removeArtifactSnapshot as jest.Mock).mock.invocationCallOrder[0])
+      .toBeLessThan(manager.completeCleanupRecord.mock.invocationCallOrder[0]);
+  });
+
   it('persists bounded raw guest output when an audit directory is configured', async () => {
     const { manager, deps, stdin } = harness();
     const backend = createBackend(config({ auditDir: '/tmp/audit' }), deps);

src/cloud-hypervisor-runtime-backend.ts

@@ -95,6 +95,7 @@ interface CloudHypervisorManagerAdapter {
   stop(options?: { preserve?: boolean; beforeCleanup?: () => Promise<void> }): Promise<void>;
   collectDiagnostics(directory: string): Promise<void>;
   collectGuestOutputAudit(directory: string): Promise<void>;
+  completeCleanupRecord(): Promise<void>;
 }
 
 /** @internal Exposed only for unit tests — not part of the public API. */
@@ -223,6 +224,7 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
   private diagnosticsCollected = false;
   private agentExecutionStarted = false;
   private readonly failedBootDiagnostics: string[] = [];
+  private readonly cleanedManagers = new Set<CloudHypervisorManagerAdapter>();
 
   constructor(
     private readonly config: WrapperConfig,
@@ -258,7 +260,7 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
       '[cloud-hypervisor] runtime=cloud-hypervisor maturity=preview fallback=disabled',
     );
     try {
-      await this.preflight();
+      if (!this.preflightResult) await this.preflight();
       stage = 'compose-infrastructure';
       await this.dependencies.startInfrastructure(
         workDir,
@@ -552,7 +554,10 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
         new Promise<void>((resolve) => setTimeout(resolve, CLOUD_HYPERVISOR_CANCEL_GRACE_MS)),
       ]);
     }
-    await this.manager?.stop({ preserve });
+    if (this.manager) {
+      await this.manager.stop({ preserve });
+      if (!preserve) this.cleanedManagers.add(this.manager);
+    }
     if (!preserve) await this.cleanupArtifactSnapshot();
   }
 
@@ -561,6 +566,11 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
     if (!directory) return;
     await this.dependencies.removeArtifactSnapshot(directory);
     this.preflightResult = undefined;
+    if (this.manager) this.cleanedManagers.add(this.manager);
+    for (const manager of this.cleanedManagers) {
+      await manager.completeCleanupRecord();
+    }
+    this.cleanedManagers.clear();
   }
 
   private async probeGuestConnectivity(bootAttempt: number): Promise<void> {
@@ -799,6 +809,7 @@ class CloudHypervisorRuntimeBackend implements ExternalAgentRuntimeBackend {
     };
     try {
       await manager.stop({ beforeCleanup: collectPreCleanupDiagnostics });
+      this.cleanedManagers.add(manager);
       this.manager = undefined;
       this.environment = undefined;
     } catch (cleanupError) {