Use instance spec compatibility checks in migration sync phase (#169)

Stash an instance's spec in its server context. During a migration, have the source send its spec to the target, and have the target use is_migration_compatible to check the specs for compatibility.

Author: gjcolombo

bin/propolis-server/src/lib/migrate/destination.rs

@@ -4,6 +4,7 @@ use hyper::upgrade::Upgraded;
 use propolis::common::GuestAddr;
 use propolis::instance::MigrateRole;
 use propolis::migrate::{MigrateStateError, Migrator};
+use propolis_client::instance_spec::MigrationCompatible;
 use slog::{error, info, warn};
 use std::io;
 use std::sync::Arc;
@@ -86,12 +87,14 @@ impl DestinationProtocol {
                 Err(MigrateError::UnexpectedMessage)
             }
         }?;
-        info!(self.log(), "Src read Preamble: {:?}", preamble);
-        // XXX: For demonstration purposes only.
-        if preamble.vm_descr.vcpus != vec![0u32, 1, 2, 3] {
+        info!(self.log(), "Destination read Preamble: {:?}", preamble);
+        if !preamble
+            .instance_spec
+            .is_migration_compatible(&self.mctx.instance_spec)
+        {
             error!(
                 self.log(),
-                "invalid CPU count in preamble ({:?})", preamble.vm_descr.vcpus
+                "Source and destination instance specs incompatible"
             );
             return Err(MigrateError::InvalidInstanceState);
         }
@@ -216,6 +219,11 @@ impl DestinationProtocol {
 
         let inv = self.mctx.instance.inv();
         for device in devices {
+            info!(
+                self.log(),
+                "Applying state to device {}", device.instance_name
+            );
+
             let dev_ent =
                 inv.get_by_name(&device.instance_name).ok_or_else(|| {
                     MigrateError::UnknownDevice(device.instance_name.clone())

bin/propolis-server/src/lib/migrate/mod.rs

@@ -8,7 +8,10 @@ use propolis::{
     instance::{Instance, MigratePhase, MigrateRole, State, TransitionError},
     migrate::MigrateStateError,
 };
-use propolis_client::api::{self, MigrationState};
+use propolis_client::{
+    api::{self, MigrationState},
+    instance_spec::InstanceSpec,
+};
 use serde::{Deserialize, Serialize};
 use slog::{error, info, o};
 use thiserror::Error;
@@ -60,6 +63,9 @@ pub struct MigrateContext {
     /// A handle to the underlying propolis [`Instance`].
     instance: Arc<Instance>,
 
+    /// A copy of this instance's spec.
+    instance_spec: InstanceSpec,
+
     /// Async descriptor context for the migrate task to access machine state in async context.
     async_ctx: AsyncCtx,
 
@@ -71,13 +77,15 @@ impl MigrateContext {
     fn new(
         migration_id: Uuid,
         instance: Arc<Instance>,
+        instance_spec: InstanceSpec,
         log: slog::Logger,
     ) -> MigrateContext {
         MigrateContext {
             migration_id,
             state: RwLock::new(MigrationState::Sync),
             async_ctx: instance.async_ctx(),
             instance,
+            instance_spec,
             log,
         }
     }
@@ -311,6 +319,7 @@ pub async fn source_start(
     let migrate_context = Arc::new(MigrateContext::new(
         migration_id,
         context.instance.clone(),
+        context.spec.clone(),
         log.clone(),
     ));
     let mctx = migrate_context.clone();
@@ -434,6 +443,7 @@ pub async fn dest_initiate(
     let migrate_context = Arc::new(MigrateContext::new(
         migration_id,
         context.instance.clone(),
+        context.spec.clone(),
         log.clone(),
     ));
     let mctx = migrate_context.clone();

bin/propolis-server/src/lib/migrate/preamble.rs

@@ -1,75 +1,14 @@
-use propolis::instance::Instance;
+use propolis_client::instance_spec::InstanceSpec;
 use serde::{Deserialize, Serialize};
 
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) enum MemType {
-    RAM,
-    ROM,
-    Dev,
-    Res,
-}
-
-// Memory regions in the guest physical address space.
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) struct MemRegion {
-    pub start: u64,
-    pub end: u64,
-    pub typ: MemType,
-}
-
-// PCI vendor and device ID pairs.
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) struct PciId {
-    pub vendor: u16,
-    pub device: u16,
-}
-
-// PCI Bus/Device/Function.
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) struct PciBdf {
-    pub bus: u16,
-    pub device: u8,
-    pub function: u8,
-}
-
-// PIO ranges associated with some device.
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) struct DevPorts {
-    pub device: u32,
-    pub ports: Vec<u16>,
-}
-
-#[derive(Deserialize, Serialize, Debug)]
-pub(crate) struct VmDescr {
-    pub vcpus: Vec<u32>,           // APIC IDs.
-    pub ioapics: Vec<u32>,         // IOAPICs.
-    pub mem: Vec<MemRegion>,       // Start, end, type
-    pub pci: Vec<(PciId, PciBdf)>, // Vendor/ID + BDF
-    pub ports: Vec<DevPorts>,
-}
-
-impl VmDescr {
-    pub fn new(_instance: &Instance) -> VmDescr {
-        // XXX: Just for demo purposes. We should get this from the instance.
-        let vcpus = vec![0, 1, 2, 3];
-        VmDescr {
-            vcpus,
-            ioapics: Vec::new(),
-            mem: Vec::new(),
-            pci: Vec::new(),
-            ports: Vec::new(),
-        }
-    }
-}
-
 #[derive(Deserialize, Serialize, Debug)]
 pub(crate) struct Preamble {
-    pub vm_descr: VmDescr,
+    pub instance_spec: InstanceSpec,
     pub blobs: Vec<Vec<u8>>,
 }
 
 impl Preamble {
-    pub fn new(instance: &Instance) -> Preamble {
-        Preamble { vm_descr: VmDescr::new(instance), blobs: Vec::new() }
+    pub fn new(instance_spec: InstanceSpec) -> Preamble {
+        Preamble { instance_spec, blobs: Vec::new() }
     }
 }

bin/propolis-server/src/lib/migrate/source.rs

@@ -81,7 +81,7 @@ impl SourceProtocol {
 
     async fn sync(&mut self) -> Result<(), MigrateError> {
         self.mctx.set_state(MigrationState::Sync).await;
-        let preamble = Preamble::new(self.mctx.instance.as_ref());
+        let preamble = Preamble::new(self.mctx.instance_spec.clone());
         let s = ron::ser::to_string(&preamble)
             .map_err(codec::ProtocolError::from)?;
         self.send_msg(codec::Message::Serialized(s)).await?;

bin/propolis-server/src/lib/server.rs

@@ -87,6 +87,7 @@ pub(crate) struct InstanceContext {
     // The instance, which may or may not be instantiated.
     pub instance: Arc<Instance>,
     pub properties: api::InstanceProperties,
+    pub spec: instance_spec::InstanceSpec,
     serial: Option<Arc<Serial<LpcUart>>>,
     state_watcher: watch::Receiver<StateChange>,
     serial_task: Option<SerialTask>,
@@ -448,6 +449,7 @@ async fn instance_ensure(
     *context = Some(InstanceContext {
         instance: instance.clone(),
         properties,
+        spec,
         serial: com1,
         state_watcher: rx,
         serial_task: None,