Add the second part of the multirack join service

clients/bootstrap-agent-lockstep-client/src/lib.rs

@@ -41,6 +41,8 @@ progenitor::generate_api!(
         RouterLifetimeConfig = sled_agent_types::early_networking::RouterLifetimeConfig,
         RssStep = bootstrap_agent_lockstep_types::RssStep,
         ServiceIpPoolConfig = bootstrap_agent_lockstep_types::ServiceIpPoolConfig,
+        SledAgentInfo = bootstrap_agent_lockstep_types::SledAgentInfo,
+        StartSledAgentStatus = bootstrap_agent_lockstep_types::StartSledAgentStatus,
         SwitchSlot = sled_agent_types::early_networking::SwitchSlot,
         TxEqConfig = sled_agent_types::early_networking::TxEqConfig,
         UplinkAddressConfig = sled_agent_types::early_networking::UplinkAddressConfig,

openapi/bootstrap-agent-lockstep.json

@@ -834,6 +834,18 @@
           "last"
         ]
       },
+      "Ipv6Subnet": {
+        "description": "Wraps an [`Ipv6Net`] with a compile-time prefix length.",
+        "type": "object",
+        "properties": {
+          "net": {
+            "$ref": "#/components/schemas/Ipv6Net"
+          }
+        },
+        "required": [
+          "net"
+        ]
+      },
       "LinkFec": {
         "description": "The forward error correction mode of a link.",
         "oneOf": [
@@ -1210,6 +1222,39 @@
               "transient_errors"
             ]
           },
+          {
+            "type": "object",
+            "properties": {
+              "sleds": {
+                "title": "BiHashMap",
+                "x-rust-type": {
+                  "crate": "iddqd",
+                  "parameters": [
+                    {
+                      "$ref": "#/components/schemas/SledAgentInfo"
+                    }
+                  ],
+                  "path": "iddqd::BiHashMap",
+                  "version": "*"
+                },
+                "type": "array",
+                "items": {
+                  "$ref": "#/components/schemas/SledAgentInfo"
+                },
+                "uniqueItems": true
+              },
+              "state": {
+                "type": "string",
+                "enum": [
+                  "start_sled_agents"
+                ]
+              }
+            },
+            "required": [
+              "sleds",
+              "state"
+            ]
+          },
           {
             "type": "object",
             "properties": {
@@ -2137,6 +2182,43 @@
           "ranges"
         ]
       },
+      "SledAgentInfo": {
+        "description": "Status information for a given sled agent",
+        "type": "object",
+        "properties": {
+          "baseboard_id": {
+            "$ref": "#/components/schemas/BaseboardId"
+          },
+          "fatal_error": {
+            "nullable": true,
+            "type": "string"
+          },
+          "sled_id": {
+            "$ref": "#/components/schemas/SledUuid"
+          },
+          "sled_subnet": {
+            "$ref": "#/components/schemas/Ipv6Subnet"
+          },
+          "started": {
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "baseboard_id",
+          "sled_id",
+          "sled_subnet",
+          "started"
+        ]
+      },
+      "SledUuid": {
+        "x-rust-type": {
+          "crate": "omicron-uuid-kinds",
+          "path": "omicron_uuid_kinds::SledUuid",
+          "version": "*"
+        },
+        "type": "string",
+        "format": "uuid"
+      },
       "SwitchSlot": {
         "description": "Identifies switch physical location",
         "oneOf": [

sled-agent/bootstrap-agent-lockstep-api/Cargo.toml

@@ -12,4 +12,3 @@ dropshot.workspace = true
 bootstrap-agent-lockstep-types.workspace = true
 omicron-uuid-kinds.workspace = true
 omicron-workspace-hack.workspace = true
-sled-agent-multirack-join.workspace = true

sled-agent/bootstrap-agent-lockstep-api/src/lib.rs

@@ -11,13 +11,13 @@
 
 use bootstrap_agent_lockstep_types::BaseboardIds;
 use bootstrap_agent_lockstep_types::MultirackJoinRequest;
+use bootstrap_agent_lockstep_types::MultirackJoinServiceState;
 use bootstrap_agent_lockstep_types::RackInitializeRequest;
 use bootstrap_agent_lockstep_types::RackOperationStatus;
 use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig;
 use dropshot::{HttpError, HttpResponseOk, RequestContext, TypedBody};
 use omicron_uuid_kinds::MultirackJoinUuid;
 use omicron_uuid_kinds::RackInitUuid;
-use sled_agent_multirack_join::MultirackJoinServiceState;
 
 #[dropshot::api_description]
 pub trait BootstrapAgentLockstepApi {

sled-agent/bootstrap-agent-lockstep-types/src/lib.rs

@@ -13,6 +13,7 @@ use anyhow::Context as _;
 use iddqd::IdOrdItem;
 use iddqd::IdOrdMap;
 use iddqd::id_upcast;
+use iddqd::{BiHashItem, BiHashMap, bi_upcast};
 use omicron_common::address::AZ_PREFIX_LENGTH;
 use omicron_common::address::IpRange;
 use omicron_common::address::IpVersion;
@@ -27,16 +28,23 @@ use omicron_common::api::external::UserId;
 use omicron_common::api::internal::nexus::Certificate;
 use omicron_uuid_kinds::MultirackJoinUuid;
 use omicron_uuid_kinds::RackInitUuid;
+use omicron_uuid_kinds::RackUuid;
+use omicron_uuid_kinds::SledUuid;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use sled_agent_types::early_networking::RackNetworkConfig;
 use sled_hardware_types::BaseboardId;
+use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::net::IpAddr;
 use std::net::Ipv6Addr;
 use strum::EnumCount;
 use strum::EnumIter;
 use strum::IntoEnumIterator;
+use trust_quorum_types::messages::ReconfigureMsg as TqReconfigureMsg;
+use trust_quorum_types::status::CoordinatorStatus;
+use trust_quorum_types::types::Epoch;
+use trust_quorum_types::types::Threshold;
 
 /// Configuration for the "rack setup service".
 ///
@@ -532,3 +540,89 @@ impl IdOrdItem for BootstrapIpOfBaseboardId {
 pub struct BaseboardIds {
     pub data: IdOrdMap<BootstrapIpOfBaseboardId>,
 }
+
+/// The state of the commit phase of the trust quorum protocol
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct CommitState {
+    pub rack_id: RackUuid,
+    pub members: BTreeSet<BaseboardId>,
+    pub epoch: Epoch,
+    pub last_committed_epoch: Option<Epoch>,
+    pub threshold: Threshold,
+    pub commit_crash_tolerance: u8,
+    pub acked: BTreeSet<BaseboardId>,
+    pub fatal_errors: BTreeMap<BaseboardId, String>,
+    pub transient_errors: BTreeMap<BaseboardId, String>,
+}
+
+/// Status information for a given sled agent
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct SledAgentInfo {
+    pub baseboard_id: BaseboardId,
+    pub sled_id: SledUuid,
+    pub sled_subnet: Ipv6Subnet<SLED_PREFIX_LENGTH>,
+    pub started: bool,
+    pub fatal_error: Option<String>,
+}
+
+impl BiHashItem for SledAgentInfo {
+    type K1<'a> = &'a BaseboardId;
+    type K2<'a> = &'a SledUuid;
+
+    fn key1(&self) -> Self::K1<'_> {
+        &self.baseboard_id
+    }
+
+    fn key2(&self) -> Self::K2<'_> {
+        &self.sled_id
+    }
+
+    bi_upcast!();
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct StartSledAgentsStatus {
+    pub sleds: BiHashMap<SledAgentInfo>,
+}
+
+impl StartSledAgentsStatus {
+    pub fn new(req: MultirackJoinRequest) -> Self {
+        let rack_subnet = Ipv6Subnet::<RACK_PREFIX_LENGTH>::new(
+            req.rack_network_config.rack_subnet.addr(),
+        );
+        let sleds = req
+            .trust_quorum_peers
+            .into_iter()
+            .enumerate()
+            .map(|(idx, baseboard_id)| SledAgentInfo {
+                baseboard_id,
+                sled_id: SledUuid::new_v4(),
+                sled_subnet: get_64_subnet(
+                    rack_subnet,
+                    u8::try_from(idx + 1).expect("too many sleds"),
+                ),
+                started: false,
+                fatal_error: None,
+            })
+            .collect();
+
+        StartSledAgentsStatus { sleds }
+    }
+}
+
+/// The current state of the `MultirackJoinService` as retrieved from the
+/// `output_rx` watch channel.
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case", tag = "state")]
+pub enum MultirackJoinServiceState {
+    Uninitialized,
+    Requested,
+    Starting,
+    TrustQuorumReconfigure(TqReconfigureMsg),
+    TrustQuorumPreparing(CoordinatorStatus),
+    TrustQuorumCommitting(CommitState),
+    StartSledAgents(StartSledAgentsStatus),
+    Completed,
+    Failed { message: String },
+    TaskPanicked,
+}

sled-agent/bootstrap-common/Cargo.toml

@@ -11,14 +11,20 @@ workspace = true
 [dependencies]
 bootstore.workspace = true
 camino.workspace = true
+omicron-common.workspace = true
 omicron-ledger.workspace = true
+omicron-uuid-kinds.workspace = true
 serde.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }
 sled-agent-config-reconciler.workspace = true
 sled-agent-measurements.workspace = true
+sled-agent-types.workspace = true
 slog-error-chain.workspace = true
 slog.workspace = true
 sprockets-tls.workspace = true
 thiserror.workspace = true
+tokio.workspace = true
 trust-quorum.workspace = true
+uuid.workspace = true
 
 omicron-workspace-hack.workspace = true

sled-agent/bootstrap-common/src/lib.rs

@@ -8,6 +8,8 @@
 //! and the Multirack Join Service. Please do not use it in public facing,
 //! non-lockstep, interfaces.
 
+pub mod sprockets;
+
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use omicron_ledger::{Ledger, Ledgerable};