feat(supervisor): Phase 2 - gRPC wire protocol + two-service packaging

.github/workflows/build-deb-package-and-integration-tests.yml

@@ -211,6 +211,11 @@ jobs:
           # picks up DigitalOcean's VPC-internal resolver (e.g. 10.110.15.254),
           # which is unreachable from inside the VMs and breaks all guest DNS.
           echo 'ALEPH_VM_DNS_NAMESERVERS=["1.1.1.1","8.8.8.8"]' >> supervisor.env
+          # This file replaces the packaged supervisor.env (kept via --force-confold),
+          # so it must carry the two-service socket the package ships; otherwise the
+          # agent runs embedded alongside the daemon (two pools fighting over the VM
+          # tap -> "File descriptor in bad state"). Matches packaging/aleph-vm/etc/aleph-vm/supervisor.env.
+          echo ALEPH_VM_SUPERVISOR_GRPC_SOCKET=/var/lib/aleph/vm/supervisor.sock >> supervisor.env
           ssh  root@${DROPLET_IPV4} mkdir -p /etc/aleph-vm/
           scp supervisor.env root@${DROPLET_IPV4}:/etc/aleph-vm/supervisor.env
           scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt
@@ -241,6 +246,10 @@ jobs:
         run: |
           ssh root@${DROPLET_IPV4} "systemctl status aleph-vm-supervisor --no-pager" || true
           ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor -n 100 --no-pager" || true
+          # Two-service split: the HTTP API (port 4020, /control/*) is served by
+          # the agent, so capture its status and journal too.
+          ssh root@${DROPLET_IPV4} "systemctl status aleph-vm-agent --no-pager" || true
+          ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-agent -n 200 --no-pager" || true
 
       - name: "Test runtime: Debian 12, SDK 0.9.0"
         run: ./.github/scripts/test_runtime_on_droplet.sh "${DROPLET_IPV4}" "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace"
@@ -262,6 +271,8 @@ jobs:
         if: ${{ !cancelled() && steps.system-booted.outcome == 'success'}}
         run: |
           ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor"
+          # Two-service split: /control/* (port 4020) is served by the agent.
+          ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-agent" || true
 
       - name: Cleanup
         if: always()

packaging/aleph-vm/DEBIAN/postinst

@@ -51,6 +51,12 @@ fi
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
   systemctl daemon-reload
-  systemctl enable aleph-vm-supervisor.service
+  systemctl enable aleph-vm-supervisor.service aleph-vm-agent.service
   systemctl restart aleph-vm-supervisor.service
+  # Wait for the daemon socket before the agent dials it (split mode).
+  for _ in $(seq 1 30); do
+    [ -S /var/lib/aleph/vm/supervisor.sock ] && break
+    sleep 1
+  done
+  systemctl restart aleph-vm-agent.service
 fi

packaging/aleph-vm/DEBIAN/preinst

@@ -5,11 +5,11 @@ set -uf -o pipefail
 
 # Systemd is absent from containers
 if ! [[ -v container ]]; then
-  # Stop the service during an upgrade.
-  # The service does not exist during a new install and will fail, this is okay
-  systemctl stop aleph-vm-supervisor.service
+  # Stop the services during an upgrade.
+  # The services do not exist during a new install and will fail, this is okay.
+  # Stop the agent first, then the supervisor daemon, mirroring the dependency order.
+  systemctl stop aleph-vm-agent.service 2>/dev/null || true
+  systemctl stop aleph-vm-supervisor.service 2>/dev/null || true
 fi
 
 set -e
-
-

packaging/aleph-vm/DEBIAN/prerm

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -euf -o pipefail
 
-systemctl disable aleph-vm-supervisor.service || true
-systemctl stop aleph-vm-supervisor.service || true
-systemctl reset-failed aleph-vm-supervisor.service 2>/dev/null || true
+for unit in aleph-vm-agent.service aleph-vm-supervisor.service; do
+  systemctl disable "$unit" || true
+  systemctl stop "$unit" || true
+  systemctl reset-failed "$unit" 2>/dev/null || true
+done

packaging/aleph-vm/etc/aleph-vm/supervisor.env

@@ -2,3 +2,6 @@
 ALEPH_VM_PRINT_SYSTEM_LOGS=False
 ALEPH_VM_DOMAIN_NAME=vm.example.org
 ALEPH_VM_PAYMENT_RECEIVER_ADDRESS=
+# Two-process split: the agent talks to the supervisor daemon over this socket.
+# Both units read this file; the daemon binds it, the agent dials it.
+ALEPH_VM_SUPERVISOR_GRPC_SOCKET=/var/lib/aleph/vm/supervisor.sock

packaging/aleph-vm/etc/systemd/system/aleph-vm-agent.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=Aleph.im VM agent (CRN HTTP API)
+After=network.target aleph-vm-supervisor.service
+Wants=aleph-vm-supervisor.service
+
+[Service]
+User=0
+Group=0
+WorkingDirectory=/opt/aleph-vm
+Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
+Environment=PYTHONDONTWRITEBYTECODE="enabled"
+EnvironmentFile=/etc/aleph-vm/supervisor.env
+ExecStart=python3 -m aleph.vm.orchestrator --print-settings
+Restart=on-failure
+RestartPreventExitStatus=0 130 143
+RestartSec=10s
+TimeoutStopSec=30s
+
+[Install]
+WantedBy=multi-user.target

packaging/aleph-vm/etc/systemd/system/aleph-vm-supervisor.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Aleph.im VM execution engine
+Description=Aleph.im VM supervisor daemon (pool owner, gRPC)
 After=network.target
 After=docker.service
 Wants=docker.service
@@ -11,11 +11,8 @@ WorkingDirectory=/opt/aleph-vm
 Environment=PYTHONPATH=/opt/aleph-vm/:$PYTHONPATH
 Environment=PYTHONDONTWRITEBYTECODE="enabled"
 EnvironmentFile=/etc/aleph-vm/supervisor.env
-ExecStart=python3 -m aleph.vm.orchestrator --print-settings
+ExecStart=python3 -m aleph.vm.supervisor
 Restart=on-failure
-# Numeric exit codes for portability — signal names need to be without "SIG"
-# prefix in systemd, and older systemd versions only accept numeric codes.
-# 130 = killed by SIGINT (128+2), 143 = killed by SIGTERM (128+15).
 RestartPreventExitStatus=0 130 143
 RestartSec=10s
 TimeoutStopSec=30s

proto/supervisor.proto

@@ -45,6 +45,8 @@ service Supervisor {
   rpc StartVm(StartVmRequest) returns (VmInfo);
   rpc RebootVm(RebootVmRequest) returns (VmInfo);
   rpc ReinstallVm(ReinstallVmRequest) returns (VmInfo);
+  rpc RunProgramCode(RunProgramCodeRequest) returns (RunProgramCodeResponse);
+  rpc RestoreFromImage(RestoreFromImageRequest) returns (VmInfo);
 
   // ── Port forwarding ──
   rpc AddPortForward(AddPortForwardRequest) returns (PortForwardInfo);
@@ -72,15 +74,16 @@ service Supervisor {
   rpc DeleteBackup(DeleteBackupRequest) returns (DeleteBackupResponse);
   rpc RestoreBackup(RestoreBackupRequest) returns (VmInfo);
 
-  // ── Migration ──
-  rpc ExportVm(ExportVmRequest) returns (MigrationInfo);
-  rpc ImportVm(ImportVmRequest) returns (VmInfo);
-  rpc GetMigrationStatus(GetMigrationStatusRequest) returns (MigrationInfo);
-
   // ── Confidential ──
   rpc InitializeConfidential(InitializeConfidentialRequest) returns (InitializeConfidentialResponse);
   rpc GetMeasurement(GetMeasurementRequest) returns (Measurement);
   rpc InjectSecret(InjectSecretRequest) returns (InjectSecretResponse);
+
+  // ── Network ──
+  rpc RecreateNetwork(RecreateNetworkRequest) returns (RecreateNetworkResponse);
+
+  // ── Reservation ──
+  rpc ReserveResources(ReserveResourcesRequest) returns (ReserveResourcesResponse);
 }
 
 // ── Host ─────────────────────────────────────────────────────────────────
@@ -129,6 +132,11 @@ message HostInfo {
 
   // Networking
   string host_ipv4 = 17;             // primary external IPv4 of the host; empty when host networking is disabled
+
+  // Reservation-aware figures the agent's /about endpoints surface
+  uint64 available_disk_bytes = 18;
+  string gpu_inventory_json = 19;    // list[dict] as JSON; rich agent GPU inventory
+  string available_gpus_json = 20;   // list[dict] as JSON
 }
 
 message NumaNode {
@@ -201,6 +209,7 @@ message CreateVmRequest {
   // signal. What is spoken over the channel is the client's business, opaque
   // to the supervisor.
   optional GuestChannel guest_channel = 14;
+  string owner_address = 16;         // VM owner's Aleph address; engine consumes this owner's GPU reservation
 }
 
 message GuestChannel {
@@ -245,6 +254,7 @@ message TeeConfig {
   TeeBackend backend = 1;            // attestation backend (orthogonal to the top-level Backend enum, which selects the VMM)
   string policy = 2;                 // empty = default
   string session_dir = 3;            // confidential session files
+  string firmware_path = 4;          // resolved OVMF blob path (empty = none)
 }
 
 message NetworkConfig {
@@ -254,8 +264,10 @@ message NetworkConfig {
 }
 
 message GpuConfig {
-  string pci_host = 1;
+  string pci_host = 1;               // RESOLVED concrete address; empty in a request
   bool supports_x_vga = 2;
+  string device_id = 3;              // REQUEST: vendor:device, e.g. "10de:2504"
+  string model = 4;                  // REQUEST: human label
 }
 
 // The confidential-computing mode a VM is actually running under. Precise by
@@ -354,6 +366,21 @@ message ReinstallVmRequest {
                                      // field is treated as true.
 }
 
+message RestoreFromImageRequest {
+  string vm_id = 1;
+  string image_path = 2;             // host path to a staged QCOW2 image
+  uint64 max_virtual_size_bytes = 3; // 0 = no cap
+}
+
+message RunProgramCodeRequest {
+  string vm_id = 1;
+  bytes scope_msgpack = 2;           // ASGI scope dict, msgpack-encoded
+  double timeout_secs = 3;
+}
+message RunProgramCodeResponse {
+  bytes reply = 1;                   // raw runtime reply, opaque to the supervisor
+}
+
 // ── Events ───────────────────────────────────────────────────────────────
 
 message WatchEventsRequest {}
@@ -451,11 +478,15 @@ message BackupInfo {
   uint64 size_bytes = 4;             // 0 until COMPLETE
   uint64 created_at_unix_secs = 5;
   string error_message = 6;          // populated when status = FAILED
+  string checksum = 7;                  // archive checksum, populated when COMPLETE
+  repeated string volumes = 8;          // archived volume names
+  map<string, uint64> source_sizes = 9; // per-volume uncompressed source size
 }
 
 message StartBackupRequest {
   string vm_id = 1;
   bool quiesce_guest = 2;            // request guest fs-freeze if supported
+  bool include_volumes = 3;          // also archive non-read-only persistent volumes
 }
 
 message GetBackupStatusRequest { string vm_id = 1; string backup_id = 2; }
@@ -474,48 +505,6 @@ message DeleteBackupResponse {}
 
 message RestoreBackupRequest { string vm_id = 1; string backup_id = 2; }
 
-// ── Migration ────────────────────────────────────────────────────────────
-
-enum MigrationPhase {
-  MIGRATION_PHASE_UNSPECIFIED = 0;
-  MIGRATION_PHASE_PREPARING = 1;
-  MIGRATION_PHASE_EXPORTING = 2;
-  MIGRATION_PHASE_IMPORTING = 3;
-  MIGRATION_PHASE_COMPLETE = 4;
-  MIGRATION_PHASE_FAILED = 5;
-}
-
-message MigrationInfo {
-  string vm_id = 1;
-  string migration_id = 2;
-  MigrationPhase phase = 3;
-  uint64 bytes_transferred = 4;
-  uint64 bytes_total = 5;
-  string error_message = 6;
-}
-
-// NOTE (Plan 0.A): the directory-based shape below is provisional.
-// aleph-vm's current migration is HTTP-based (the source exposes
-// /control/machine/{ref}/migration/disk/... and the destination
-// fetches). The contract needs reshaping for host-to-host transport
-// before Plan 0.C wires real implementations. See design doc §9 open
-// questions.
-// Phase 1 removed the directory-based migration; drop these RPCs in the Phase 2 proto pass.
-message ExportVmRequest {
-  string vm_id = 1;
-  string destination_dir = 2;        // PROVISIONAL: local path on the host
-}
-
-message ImportVmRequest {
-  string vm_id = 1;                  // id the new VM will be assigned post-import
-  string source_dir = 2;             // PROVISIONAL: see note on ExportVmRequest
-}
-
-message GetMigrationStatusRequest {
-  string vm_id = 1;
-  string migration_id = 2;
-}
-
 // ── Confidential ─────────────────────────────────────────────────────────
 
 message InitializeConfidentialRequest {
@@ -528,10 +517,22 @@ message InitializeConfidentialResponse {}
 
 message GetMeasurementRequest { string vm_id = 1; }
 
+message SevInfo {
+  bool enabled = 1;
+  uint32 api_major = 2;
+  uint32 api_minor = 3;
+  uint32 build_id = 4;
+  uint32 policy = 5;
+  string state = 6;
+  uint32 handle = 7;
+}
+
 message Measurement {
   string vm_id = 1;
   bytes measurement_bytes = 2;       // attestation report / SEV launch measure
   TeeBackend tee_backend = 3;
+  SevInfo sev_info = 4;              // present for SEV launches
+  string launch_measure = 5;        // base64 launch measurement
 }
 
 message InjectSecretRequest {
@@ -542,6 +543,27 @@ message InjectSecretRequest {
 
 message InjectSecretResponse {}
 
+// ── Network ──────────────────────────────────────────────────────────────
+
+message RecreateNetworkRequest {}
+message RecreateNetworkResponse {
+  string summary_json = 1;           // JSON-encoded summary dict from the engine
+}
+
+// ── Reservation ──────────────────────────────────────────────────────────
+
+message ReserveResourcesRequest {
+  string user_address = 1;
+  uint32 vcpus = 2;
+  uint64 memory_mib = 3;
+  uint64 disk_mib = 4;
+  bool is_instance = 5;
+  repeated GpuConfig gpus = 6;        // request: matched by device_id
+}
+message ReserveResourcesResponse {
+  int64 expiry_unix_ns = 1;          // reservation expiry, unix ns UTC
+}
+
 // ── Wire error vocabulary ────────────────────────────────────────────────
 //
 // Closed enum of errors the supervisor can return. Server side: map

src/aleph/vm/controllers/firecracker/spec_program.py

@@ -62,6 +62,17 @@ def from_spec(cls, spec: CreateVmSpec) -> SpecProgramResources:
             extra_disks=[disk for disk in spec.disks if disk.role is DiskRole.EXTRA],
         )
 
+    def get_disk_usage_delta(self) -> int:
+        """Disk reserved-but-unused by this execution, summed by the pool's
+        capacity admission (calculate_available_disk) across all executions.
+
+        Spec disk admission is deferred: DiskSpec carries no size yet (the
+        program rootfs is a read-only squashfs and extra disks declare no size),
+        so a spec program reserves nothing. Mirrors check_spec_admission passing
+        disk_mib=0. Returning 0 keeps SpecProgramResources usable in the shared
+        admission path that the other resource types implement."""
+        return 0
+
     def to_dict(self):
         return self.__dict__
 

src/aleph/vm/models.py

@@ -152,7 +152,7 @@ async def fetch_port_redirect_config_and_setup(self):
         """Fetch the user's port-forwarding aggregate and apply updates.
 
         Persisted-mapping reload is the creator's job
-        (pool.create_a_vm / create_vm_from_spec / restart_persistent_vm).
+        (pool.create_vm_from_spec / restart_persistent_vm).
         """
         if not self.is_instance:
             return

src/aleph/vm/orchestrator/views/__init__.py

@@ -79,6 +79,7 @@
     SupervisorError,
     VmNotFoundError,
 )
+from aleph.vm.supervisor.translate import build_reservation_request
 from aleph.vm.supervisor.types import (
     ConfidentialMode,
     PortForwardInfo,
@@ -831,8 +832,8 @@ async def notify_allocation(request: web.Request):
     update_watcher = request.app["update_watcher"]
 
     # Capacity admission is no longer checked here: the engine enforces it
-    # atomically inside the create path (pool.check_admission for the message
-    # path, pool.check_spec_admission for the spec path), raising the typed
+    # atomically inside the create path (pool.check_spec_admission, on the
+    # shared pool.check_capacity core), raising the typed
     # InsufficientResourcesError. The vm_creation_exceptions / 503 path below
     # surfaces that error to the caller.
 
@@ -1007,10 +1008,13 @@ async def operate_reserve_resources(request: web.Request, authenticated_sender:
     except ValidationError as error:
         return web.json_response(data=error.json(), status=web.HTTPBadRequest.status_code)
 
-    # The supervisor runs capacity admission (keeping the dry-run honest) then
-    # holds the requested resources, returning the reservation expiry.
+    # The agent translates the message into a message-free resources DTO; the
+    # supervisor runs capacity admission (keeping the dry-run honest) then holds
+    # the requested resources, returning the reservation expiry. No Aleph message
+    # crosses the supervisor boundary.
+    reservation_request = build_reservation_request(message, authenticated_sender)
     try:
-        expiration_date = await supervisor.reserve_resources(message, authenticated_sender)
+        expiration_date = await supervisor.reserve_resources(reservation_request)
     except BoundaryInsufficientResourcesError as error:
         logger.warning("Refusing resource reservation: %s", error)
         return web.HTTPServiceUnavailable(