diff --git a/Cargo.lock b/Cargo.lock index 98ac5b8a015..dce97ddfcdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -925,7 +925,6 @@ dependencies = [ "dropshot", "omicron-uuid-kinds", "omicron-workspace-hack", - "sled-agent-multirack-join", ] [[package]] @@ -14008,16 +14007,22 @@ version = "0.1.0" dependencies = [ "bootstore", "camino", + "omicron-common", "omicron-ledger", + "omicron-uuid-kinds", "omicron-workspace-hack", "serde", + "serde_json", "sled-agent-config-reconciler", "sled-agent-measurements", + "sled-agent-types", "slog", "slog-error-chain", "sprockets-tls", "thiserror 2.0.18", + "tokio", "trust-quorum", + "uuid", ] [[package]] @@ -14180,6 +14185,7 @@ dependencies = [ "bootstrap-agent-lockstep-types", "camino", "camino-tempfile", + "iddqd", "itertools 0.14.0", "nexus-types", "omicron-common", diff --git a/clients/bootstrap-agent-lockstep-client/src/lib.rs b/clients/bootstrap-agent-lockstep-client/src/lib.rs index 822152fcb9a..46dc75824a3 100644 --- a/clients/bootstrap-agent-lockstep-client/src/lib.rs +++ b/clients/bootstrap-agent-lockstep-client/src/lib.rs @@ -41,6 +41,8 @@ progenitor::generate_api!( RouterLifetimeConfig = sled_agent_types::early_networking::RouterLifetimeConfig, RssStep = bootstrap_agent_lockstep_types::RssStep, ServiceIpPoolConfig = bootstrap_agent_lockstep_types::ServiceIpPoolConfig, + SledAgentInfo = bootstrap_agent_lockstep_types::SledAgentInfo, + StartSledAgentStatus = bootstrap_agent_lockstep_types::StartSledAgentStatus, SwitchSlot = sled_agent_types::early_networking::SwitchSlot, TxEqConfig = sled_agent_types::early_networking::TxEqConfig, UplinkAddressConfig = sled_agent_types::early_networking::UplinkAddressConfig, diff --git a/openapi/bootstrap-agent-lockstep.json b/openapi/bootstrap-agent-lockstep.json index 28bd3c03799..0c466f37481 100644 --- a/openapi/bootstrap-agent-lockstep.json +++ b/openapi/bootstrap-agent-lockstep.json @@ -834,6 +834,18 @@ "last" ] }, + "Ipv6Subnet": { + "description": "Wraps an [`Ipv6Net`] with a compile-time prefix length.", + "type": "object", + "properties": { + "net": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + "required": [ + "net" + ] + }, "LinkFec": { "description": "The forward error correction mode of a link.", "oneOf": [ @@ -1210,6 +1222,39 @@ "transient_errors" ] }, + { + "type": "object", + "properties": { + "sleds": { + "title": "BiHashMap", + "x-rust-type": { + "crate": "iddqd", + "parameters": [ + { + "$ref": "#/components/schemas/SledAgentInfo" + } + ], + "path": "iddqd::BiHashMap", + "version": "*" + }, + "type": "array", + "items": { + "$ref": "#/components/schemas/SledAgentInfo" + }, + "uniqueItems": true + }, + "state": { + "type": "string", + "enum": [ + "start_sled_agents" + ] + } + }, + "required": [ + "sleds", + "state" + ] + }, { "type": "object", "properties": { @@ -2137,6 +2182,43 @@ "ranges" ] }, + "SledAgentInfo": { + "description": "Status information for a given sled agent", + "type": "object", + "properties": { + "baseboard_id": { + "$ref": "#/components/schemas/BaseboardId" + }, + "fatal_error": { + "nullable": true, + "type": "string" + }, + "sled_id": { + "$ref": "#/components/schemas/SledUuid" + }, + "sled_subnet": { + "$ref": "#/components/schemas/Ipv6Subnet" + }, + "started": { + "type": "boolean" + } + }, + "required": [ + "baseboard_id", + "sled_id", + "sled_subnet", + "started" + ] + }, + "SledUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::SledUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, "SwitchSlot": { "description": "Identifies switch physical location", "oneOf": [ diff --git a/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml b/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml index f188cf69f63..59a0543706a 100644 --- a/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml +++ b/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml @@ -12,4 +12,3 @@ dropshot.workspace = true bootstrap-agent-lockstep-types.workspace = true omicron-uuid-kinds.workspace = true omicron-workspace-hack.workspace = true -sled-agent-multirack-join.workspace = true diff --git a/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs index 1bd2982244e..05a6ba807fe 100644 --- a/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs @@ -11,13 +11,13 @@ use bootstrap_agent_lockstep_types::BaseboardIds; use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use bootstrap_agent_lockstep_types::MultirackJoinServiceState; use bootstrap_agent_lockstep_types::RackInitializeRequest; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig; use dropshot::{HttpError, HttpResponseOk, RequestContext, TypedBody}; use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; -use sled_agent_multirack_join::MultirackJoinServiceState; #[dropshot::api_description] pub trait BootstrapAgentLockstepApi { diff --git a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs index f5c6e5a085b..d940b7e123e 100644 --- a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs @@ -13,6 +13,7 @@ use anyhow::Context as _; use iddqd::IdOrdItem; use iddqd::IdOrdMap; use iddqd::id_upcast; +use iddqd::{BiHashItem, BiHashMap, bi_upcast}; use omicron_common::address::AZ_PREFIX_LENGTH; use omicron_common::address::IpRange; use omicron_common::address::IpVersion; @@ -27,16 +28,23 @@ use omicron_common::api::external::UserId; use omicron_common::api::internal::nexus::Certificate; use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; +use omicron_uuid_kinds::RackUuid; +use omicron_uuid_kinds::SledUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_agent_types::early_networking::RackNetworkConfig; use sled_hardware_types::BaseboardId; +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::net::IpAddr; use std::net::Ipv6Addr; use strum::EnumCount; use strum::EnumIter; use strum::IntoEnumIterator; +use trust_quorum_types::messages::ReconfigureMsg as TqReconfigureMsg; +use trust_quorum_types::status::CoordinatorStatus; +use trust_quorum_types::types::Epoch; +use trust_quorum_types::types::Threshold; /// Configuration for the "rack setup service". /// @@ -532,3 +540,89 @@ impl IdOrdItem for BootstrapIpOfBaseboardId { pub struct BaseboardIds { pub data: IdOrdMap, } + +/// The state of the commit phase of the trust quorum protocol +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct CommitState { + pub rack_id: RackUuid, + pub members: BTreeSet, + pub epoch: Epoch, + pub last_committed_epoch: Option, + pub threshold: Threshold, + pub commit_crash_tolerance: u8, + pub acked: BTreeSet, + pub fatal_errors: BTreeMap, + pub transient_errors: BTreeMap, +} + +/// Status information for a given sled agent +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct SledAgentInfo { + pub baseboard_id: BaseboardId, + pub sled_id: SledUuid, + pub sled_subnet: Ipv6Subnet, + pub started: bool, + pub fatal_error: Option, +} + +impl BiHashItem for SledAgentInfo { + type K1<'a> = &'a BaseboardId; + type K2<'a> = &'a SledUuid; + + fn key1(&self) -> Self::K1<'_> { + &self.baseboard_id + } + + fn key2(&self) -> Self::K2<'_> { + &self.sled_id + } + + bi_upcast!(); +} + +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct StartSledAgentsStatus { + pub sleds: BiHashMap, +} + +impl StartSledAgentsStatus { + pub fn new(req: MultirackJoinRequest) -> Self { + let rack_subnet = Ipv6Subnet::::new( + req.rack_network_config.rack_subnet.addr(), + ); + let sleds = req + .trust_quorum_peers + .into_iter() + .enumerate() + .map(|(idx, baseboard_id)| SledAgentInfo { + baseboard_id, + sled_id: SledUuid::new_v4(), + sled_subnet: get_64_subnet( + rack_subnet, + u8::try_from(idx + 1).expect("too many sleds"), + ), + started: false, + fatal_error: None, + }) + .collect(); + + StartSledAgentsStatus { sleds } + } +} + +/// The current state of the `MultirackJoinService` as retrieved from the +/// `output_rx` watch channel. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case", tag = "state")] +pub enum MultirackJoinServiceState { + Uninitialized, + Requested, + Starting, + TrustQuorumReconfigure(TqReconfigureMsg), + TrustQuorumPreparing(CoordinatorStatus), + TrustQuorumCommitting(CommitState), + StartSledAgents(StartSledAgentsStatus), + Completed, + Failed { message: String }, + TaskPanicked, +} diff --git a/sled-agent/bootstrap-common/Cargo.toml b/sled-agent/bootstrap-common/Cargo.toml index 9368fd209b1..b886631c98b 100644 --- a/sled-agent/bootstrap-common/Cargo.toml +++ b/sled-agent/bootstrap-common/Cargo.toml @@ -11,14 +11,20 @@ workspace = true [dependencies] bootstore.workspace = true camino.workspace = true +omicron-common.workspace = true omicron-ledger.workspace = true +omicron-uuid-kinds.workspace = true serde.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } sled-agent-config-reconciler.workspace = true sled-agent-measurements.workspace = true +sled-agent-types.workspace = true slog-error-chain.workspace = true slog.workspace = true sprockets-tls.workspace = true thiserror.workspace = true +tokio.workspace = true trust-quorum.workspace = true +uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/sled-agent/bootstrap-common/src/lib.rs b/sled-agent/bootstrap-common/src/lib.rs index d1d85b8734b..49af67e7438 100644 --- a/sled-agent/bootstrap-common/src/lib.rs +++ b/sled-agent/bootstrap-common/src/lib.rs @@ -8,6 +8,8 @@ //! and the Multirack Join Service. Please do not use it in public facing, //! non-lockstep, interfaces. +pub mod sprockets; + use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use omicron_ledger::{Ledger, Ledgerable}; diff --git a/sled-agent/src/bootstrap/sprockets_client.rs b/sled-agent/bootstrap-common/src/sprockets.rs similarity index 72% rename from sled-agent/src/bootstrap/sprockets_client.rs rename to sled-agent/bootstrap-common/src/sprockets.rs index b4f03b0740c..d066c3e83b8 100644 --- a/sled-agent/src/bootstrap/sprockets_client.rs +++ b/sled-agent/bootstrap-common/src/sprockets.rs @@ -2,17 +2,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Interface for making API requests to a Sled Agent's Bootstrap API. - -use super::params::Request; -use super::params::RequestEnvelope; -use super::params::version; -use super::views::SledAgentResponse; -use crate::bootstrap::views::Response; -use crate::bootstrap::views::ResponseEnvelope; +//! Interface for making requests to a Sled Agent's Bootstrap API. + +use serde::{Deserialize, Serialize}; use sled_agent_measurements::{MeasurementError, MeasurementsHandle}; use sled_agent_types::sled::StartSledAgentRequest; -use slog::Logger; +use slog::{Logger, o}; use slog_error_chain::SlogInlineError; use sprockets_tls; use sprockets_tls::keys::SprocketsConfig; @@ -24,6 +19,7 @@ use thiserror::Error; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; +use uuid::Uuid; #[derive(Debug, Error, SlogInlineError)] pub enum SprocketsClientError { @@ -72,9 +68,44 @@ pub enum SprocketsClientError { MeasurementError(#[source] MeasurementError), } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum Request<'a> { + /// Send configuration information for launching a Sled Agent. + StartSledAgentRequest(Cow<'a, StartSledAgentRequest>), +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct RequestEnvelope<'a> { + pub version: u32, + pub request: Request<'a>, +} + +pub mod version { + pub const V1: u32 = 1; +} + +/// Describes the Sled Agent running on the device. +#[derive(Serialize, Deserialize, PartialEq)] +pub struct SledAgentResponse { + pub id: Uuid, +} + +#[derive(Serialize, Deserialize, PartialEq)] +// Note: We intentionally do not derive `Debug` on this type, to avoid +// accidentally debug-logging the secret share. +pub enum Response { + SledAgentResponse(SledAgentResponse), +} + +#[derive(Serialize, Deserialize, PartialEq)] +pub struct ResponseEnvelope { + pub version: u32, + pub response: Result, +} + /// A sprockets client wrapper used to connect to bootstrap agents for rack /// initialization -pub(crate) struct SprocketsClient { +pub struct SprocketsClient { addr: SocketAddrV6, log: Logger, sprockets_conf: SprocketsConfig, @@ -82,7 +113,7 @@ pub(crate) struct SprocketsClient { } impl SprocketsClient { - pub(crate) fn new( + pub fn new( addr: SocketAddrV6, sprockets_conf: SprocketsConfig, measurements: Arc, @@ -94,7 +125,7 @@ impl SprocketsClient { /// Start sled agent by sending an initialization request determined from /// RSS input. This client is on the same scrimlet as RSS, and is talking /// over TCP to all other bootstrap agents. - pub(crate) async fn start_sled_agent( + pub async fn start_sled_agent( &self, request: &StartSledAgentRequest, ) -> Result { @@ -102,7 +133,7 @@ impl SprocketsClient { Self::start_sled_agent_with_stream(stream, request).await } - pub(crate) async fn start_sled_agent_with_stream( + pub async fn start_sled_agent_with_stream( stream: sprockets_tls::Stream, request: &StartSledAgentRequest, ) -> Result { @@ -112,7 +143,7 @@ impl SprocketsClient { } } - pub(crate) async fn connect( + pub async fn connect( &self, ) -> Result, SprocketsClientError> { let log = @@ -197,3 +228,41 @@ impl SprocketsClient { envelope.response.map_err(SprocketsClientError::ServerFailure) } } + +#[cfg(test)] +mod tests { + use std::net::Ipv6Addr; + + use omicron_common::address::Ipv6Subnet; + use omicron_uuid_kinds::RackUuid; + use omicron_uuid_kinds::SledUuid; + use sled_agent_types::sled::StartSledAgentRequestBody; + + use super::*; + + #[test] + fn json_serialization_round_trips() { + let envelope = RequestEnvelope { + version: 1, + request: Request::StartSledAgentRequest(Cow::Owned( + StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: SledUuid::new_v4(), + rack_id: RackUuid::new_v4(), + use_trust_quorum: false, + is_lrtq_learner: false, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + }, + }, + )), + }; + + let serialized = serde_json::to_vec(&envelope).unwrap(); + let deserialized: RequestEnvelope = + serde_json::from_slice(serialized.as_slice()).unwrap(); + + assert!(envelope == deserialized, "serialization round trip failed"); + } +} diff --git a/sled-agent/multirack-join/Cargo.toml b/sled-agent/multirack-join/Cargo.toml index 421a6a84a0c..6bfeb4a4298 100644 --- a/sled-agent/multirack-join/Cargo.toml +++ b/sled-agent/multirack-join/Cargo.toml @@ -11,6 +11,7 @@ workspace = true bootstore.workspace = true bootstrap-agent-lockstep-types.workspace = true camino.workspace = true +iddqd.workspace = true itertools.workspace = true nexus-types.workspace = true omicron-common.workspace = true diff --git a/sled-agent/multirack-join/src/lib.rs b/sled-agent/multirack-join/src/lib.rs index 275fdced715..abdf608bbed 100644 --- a/sled-agent/multirack-join/src/lib.rs +++ b/sled-agent/multirack-join/src/lib.rs @@ -15,15 +15,24 @@ #[macro_use] extern crate slog; -use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use bootstrap_agent_lockstep_types::{ + CommitState, MultirackJoinRequest, MultirackJoinServiceState, + SledAgentInfo, StartSledAgentsStatus, +}; use nexus_types::trust_quorum::TrustQuorumConfig; +use omicron_common::address::BOOTSTRAP_AGENT_RACK_INIT_PORT; use omicron_uuid_kinds::RackUuid; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use sled_agent_bootstrap_common::sprockets::{ + SprocketsClient, SprocketsClientError, +}; use sled_agent_bootstrap_common::{RssContext, RunRssError}; +use sled_agent_types::sled::{ + StartSledAgentRequest, StartSledAgentRequestBody, +}; use sled_hardware_types::BaseboardId; -use slog::{Logger, info}; +use slog::{Logger, error, info}; use slog_error_chain::{InlineErrorChain, SlogInlineError}; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::{ collections::{BTreeMap, BTreeSet}, sync::{Arc, Mutex}, @@ -36,9 +45,7 @@ use tokio::{ }; use trust_quorum::{NodeApiError, ProxyError}; use trust_quorum_types::{ - messages::ReconfigureMsg as TqReconfigureMsg, - status::CoordinatorStatus, - types::{Epoch, Threshold}, + messages::ReconfigureMsg as TqReconfigureMsg, types::Epoch, }; const INITIAL_TRUST_QUORUM_EPOCH: Epoch = Epoch(1); @@ -67,6 +74,22 @@ pub enum MultirackJoinServiceError { #[error("Failed to join proxy commit task")] ProxyCommit(#[from] JoinError), + + #[error( + "Sprockets connections not available for all sleds. Missing {0:#?}" + )] + MissingSledConnections(BTreeSet), + + #[error("Failed to start sled-agents: {0:#?}")] + StartSledAgents(BTreeSet), +} + +#[derive(Error, Debug, SlogInlineError)] +#[error("Failed to start sled agent on {baseboard_id}")] +pub struct SledSpecificSprocketsError { + baseboard_id: BaseboardId, + #[source] + err: SprocketsClientError, } impl From for MultirackJoinServiceError { @@ -78,36 +101,6 @@ impl From for MultirackJoinServiceError { } } -/// The state of the commit phase of the trust quorum protocol -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -pub struct CommitState { - rack_id: RackUuid, - members: BTreeSet, - epoch: Epoch, - last_committed_epoch: Option, - threshold: Threshold, - commit_crash_tolerance: u8, - acked: BTreeSet, - fatal_errors: BTreeMap, - transient_errors: BTreeMap, -} - -/// The current state of the `MultirackJoinService` as retrieved from the -/// `output_rx` watch channel. -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "snake_case", tag = "state")] -pub enum MultirackJoinServiceState { - Uninitialized, - Requested, - Starting, - TrustQuorumReconfigure(TqReconfigureMsg), - TrustQuorumPreparing(CoordinatorStatus), - TrustQuorumCommitting(CommitState), - Completed, - Failed { message: String }, - TaskPanicked, -} - // The value returned from `MultirackJoinServiceTask::tq_prepare` enum TqPrepareResult { Prepared, @@ -127,6 +120,38 @@ enum TqCommitResult { }, } +/// All the information required to start a sled agent remotely over a sprockets +/// channel. +struct StartSledAgentInfo { + baseboard_id: BaseboardId, + bootstrap_ip: Ipv6Addr, + req: StartSledAgentRequest, +} + +impl StartSledAgentInfo { + fn new( + rack_id: RackUuid, + bootstrap_ip: Ipv6Addr, + info: SledAgentInfo, + ) -> Self { + StartSledAgentInfo { + baseboard_id: info.baseboard_id.clone(), + bootstrap_ip, + req: StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: info.sled_id, + subnet: info.sled_subnet, + use_trust_quorum: true, + is_lrtq_learner: false, + rack_id, + }, + }, + } + } +} + /// The interface to the Multirack Join Service. pub struct MultirackJoinServiceHandle { pub join_handle: @@ -176,8 +201,9 @@ impl MultirackJoinServiceTask { self.init_trust_quorum(rack_id).await?; + self.start_sled_agents(rack_id).await?; + // TODO: - // Start sled-agents // Configure networking // // https://github.com/oxidecomputer/omicron/issues/10637 @@ -185,6 +211,185 @@ impl MultirackJoinServiceTask { Ok(()) } + /// Try to start all sled agents on each sled with no other zones + async fn start_sled_agents( + &mut self, + rack_id: RackUuid, + ) -> Result<(), MultirackJoinServiceError> { + info!(self.log, "Starting Sled agents"); + let req = self.input_rx.borrow_and_update().clone(); + let tq_members = req.trust_quorum_peers.clone(); + let status = StartSledAgentsStatus::new(req); + self.output_tx.send_modify(|state| { + *state = MultirackJoinServiceState::StartSledAgents(status.clone()) + }); + + let mut bootstrap_ips: BTreeMap<_, _> = self + .ctx + .trust_quorum_handle + .conn_mgr_status() + .await? + .connected_peers() + .into_iter() + .collect(); + + // Insert this node into our map. Connected peers don't include ourself. + bootstrap_ips.insert( + self.ctx.trust_quorum_handle.baseboard_id().clone(), + self.ctx.global_zone_bootstrap_ip, + ); + + let mut missing_sleds = BTreeSet::new(); + for baseboard_id in tq_members { + if !bootstrap_ips.contains_key(&baseboard_id) { + missing_sleds.insert(baseboard_id); + } + } + // We have already initialized the trust quorum on all nodes at this + // point, and so the number of bootstrap ips should match our expected + // configuration. + if !missing_sleds.is_empty() { + return Err(MultirackJoinServiceError::MissingSledConnections( + missing_sleds, + )); + } + + // Attempt to start all our sled agents in parallel + let mut set = JoinSet::new(); + for info in status.sleds.iter().cloned() { + // Unwrap is safe, because we constructed both bootstrap_ips and + // status from trust_quorum_peers. + let bootstrap_ip = *bootstrap_ips.get(&info.baseboard_id).unwrap(); + self.spawn_start_sled_agent_task( + &mut set, + StartSledAgentInfo::new(rack_id, bootstrap_ip, info), + ); + } + + let mut failed = BTreeSet::new(); + + // Wait for the result of each sled-agent + while let Some(res) = set.join_next().await { + match res? { + Ok(baseboard_id) => { + info!( + self.log, + "Started sled agent"; + "baseboard_id" => %baseboard_id + ); + self.output_tx.send_modify(|state| { + let MultirackJoinServiceState::StartSledAgents(status) = + state + else { + panic!( + "MultirackJoinService in wrong state: {:#?}", + state + ); + }; + // Safe to unwrap since we only start tasks that return + // baseboards that already exist in status. + let mut info = + status.sleds.get1_mut(&baseboard_id).unwrap(); + info.started = true; + }); + } + Err(err) => { + // We already logged this error in the spawn task + self.output_tx.send_modify(|state| { + let MultirackJoinServiceState::StartSledAgents(status) = + state + else { + panic!( + "MultirackJoinService in wrong state: {:#?}", + state + ); + }; + // Safe to unwrap since we only start tasks that return + // baseboards that already exist in status. + let mut info = + status.sleds.get1_mut(&err.baseboard_id).unwrap(); + info.fatal_error = + Some(InlineErrorChain::new(&err).to_string()); + }); + failed.insert(err.baseboard_id); + } + } + } + + if failed.is_empty() { + Ok(()) + } else { + Err(MultirackJoinServiceError::StartSledAgents(failed)) + } + } + + /// Spawn a task that connects to the remote sprockets server and sends a + /// `StartSledAgentRequest`. + fn spawn_start_sled_agent_task( + &mut self, + set: &mut JoinSet>, + info: StartSledAgentInfo, + ) { + let log = self.log.new(o!( + "baseboard_id" => info.baseboard_id.to_string(), + "bootstrap_ip" => info.bootstrap_ip.to_string() + )); + + info!(log, "Attempting to start sled agent"; + ); + + let bootstrap_addr = SocketAddrV6::new( + info.bootstrap_ip, + BOOTSTRAP_AGENT_RACK_INIT_PORT, + 0, + 0, + ); + + let client = SprocketsClient::new( + bootstrap_addr, + self.ctx.sprockets_config.clone(), + self.ctx.measurements.clone(), + self.log.clone(), + ); + + set.spawn(async move { + match client.start_sled_agent(&info.req).await { + Ok(_) => Ok(info.baseboard_id), + Err(err) => { + // There really aren't any transient errors worth worrying + // about here. At this point we've already established trust + // quorum and know that we can reach each sled. We should + // be able to open another sprockets connection and start a + // sled-agent. We could backoff on failure to connect (the + // only possible transient error), but then how long do we + // wait until we give up? + // + // The client could see the transient error and then issue + // a new request with new sled membership that comes in + // on `input_rx`. We would then restart the trust quorum + // configuration. However, that's a bunch of extra code for + // a situation where the debugging of the issue by support + // is likely to take longer than doing the clean slate and + // join again. And we will want to debug this, as it should + // basically never happen, and means bootstrap network + // endpoints are becoming unavailable for some reason. + error!( + log, + "Failed to start sled agent"; + InlineErrorChain::new(&err) + ); + Err(SledSpecificSprocketsError { + baseboard_id: info.baseboard_id, + err, + }) + } + } + }); + } + + // We have already initialized the trust quorum on all nodes at this + // point, and so the number of bootstrap ips should match our expected + // configuration. /// Start initializing trust quorum given the the existing /// `MultirackJoinRequest` in input_rx. /// @@ -573,6 +778,9 @@ impl MultirackJoinServiceTask { // Check if we have received an updated membership set from an operator. // // If we have received a new set, return it. Otherwise, return `None`. + // We have already initialized the trust quorum on all nodes at this + // point, and so the number of bootstrap ips should match our expected + // configuration. // Return an error if checking for the update fails. async fn has_membership_changed( &mut self, @@ -588,3 +796,70 @@ impl MultirackJoinServiceTask { Ok(None) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + use sled_agent_types::early_networking::{ + LinkSpeed, PortConfig, RackNetworkConfig, SwitchSlot, UplinkPorts, + }; + use sled_hardware_types::BaseboardId; + + fn rack_network_config() -> RackNetworkConfig { + // RackNetworkConfig's ports must be nonempty. + let ports = vec![PortConfig { + routes: Vec::new(), + addresses: Vec::new(), + switch: SwitchSlot::Switch1, + port: "qsfp0".to_owned(), + uplink_port_speed: LinkSpeed::Speed100G, + uplink_port_fec: None, + bgp_peers: Vec::new(), + autoneg: false, + lldp: None, + tx_eq: None, + }]; + + RackNetworkConfig { + rack_subnet: "fd00:abcd:ffff::/56".parse().unwrap(), + infra_ip_first: "10.0.0.1".parse().unwrap(), + infra_ip_last: "10.0.0.100".parse().unwrap(), + ports: UplinkPorts::new(ports).unwrap(), + bgp: Vec::new(), + bfd: Vec::new(), + } + } + + fn trust_quorum_peers() -> BTreeSet { + (0..32) + .into_iter() + .map(|i| BaseboardId { + part_number: "FAKE_PART".to_string(), + serial_number: format!("2FAKE{:03}", i), + }) + .collect() + } + + #[test] + fn new_start_sled_agent_status() { + let req = MultirackJoinRequest { + trust_quorum_peers: trust_quorum_peers(), + rack_network_config: rack_network_config(), + }; + + let status = StartSledAgentsStatus::new(req.clone()); + assert_eq!(status.sleds.len(), req.trust_quorum_peers.len()); + + let actual_sled_subnets: BTreeSet<_> = + status.sleds.iter().map(|s| s.sled_subnet.to_string()).collect(); + let expected_sled_subnets: BTreeSet<_> = + (0..req.trust_quorum_peers.len()) + .into_iter() + .map(|i| format!("fd00:abcd:ffff:{:x}::/64", i + 1)) + .collect(); + + assert_eq!(actual_sled_subnets, expected_sled_subnets); + } +} diff --git a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs index 7b342d94227..beae8a4e402 100644 --- a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs +++ b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs @@ -19,6 +19,7 @@ use bootstrap_agent_lockstep_api::bootstrap_agent_lockstep_api_mod; use bootstrap_agent_lockstep_types::BaseboardIds; use bootstrap_agent_lockstep_types::BootstrapIpOfBaseboardId; use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use bootstrap_agent_lockstep_types::MultirackJoinServiceState; use bootstrap_agent_lockstep_types::RackInitializeRequest; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig; @@ -32,7 +33,6 @@ use omicron_uuid_kinds::RackInitUuid; use sled_agent_bootstrap_common::RssContext; use sled_agent_config_reconciler::InternalDisksReceiver; use sled_agent_measurements::MeasurementsHandle; -use sled_agent_multirack_join::MultirackJoinServiceState; use sled_agent_rack_setup::RackInitializeRequestParams; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; diff --git a/sled-agent/src/bootstrap/mod.rs b/sled-agent/src/bootstrap/mod.rs index 95f6d655ab3..208ac41ed40 100644 --- a/sled-agent/src/bootstrap/mod.rs +++ b/sled-agent/src/bootstrap/mod.rs @@ -7,17 +7,14 @@ pub(crate) mod bootstore_setup; mod http_entrypoints_lockstep; mod maghemite; -pub(crate) mod params; mod pre_server; mod pumpkind; mod rack_ops; pub(crate) mod rss_handle; pub mod secret_retriever; pub mod server; -pub mod sprockets_client; mod sprockets_server; pub(crate) mod trust_quorum_setup; -mod views; pub(crate) use pre_server::BootstrapNetworking; pub use rack_ops::RssAccessError; diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs deleted file mode 100644 index 82696c76f12..00000000000 --- a/sled-agent/src/bootstrap/params.rs +++ /dev/null @@ -1,63 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Request types for the bootstrap agent - -use serde::{Deserialize, Serialize}; -use sled_agent_types::sled::StartSledAgentRequest; -use std::borrow::Cow; - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum Request<'a> { - /// Send configuration information for launching a Sled Agent. - StartSledAgentRequest(Cow<'a, StartSledAgentRequest>), -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct RequestEnvelope<'a> { - pub version: u32, - pub request: Request<'a>, -} - -pub(super) mod version { - pub(crate) const V1: u32 = 1; -} - -#[cfg(test)] -mod tests { - use std::net::Ipv6Addr; - - use omicron_common::address::Ipv6Subnet; - use omicron_uuid_kinds::RackUuid; - use omicron_uuid_kinds::SledUuid; - use sled_agent_types::sled::StartSledAgentRequestBody; - - use super::*; - - #[test] - fn json_serialization_round_trips() { - let envelope = RequestEnvelope { - version: 1, - request: Request::StartSledAgentRequest(Cow::Owned( - StartSledAgentRequest { - generation: 0, - schema_version: 1, - body: StartSledAgentRequestBody { - id: SledUuid::new_v4(), - rack_id: RackUuid::new_v4(), - use_trust_quorum: false, - is_lrtq_learner: false, - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - }, - }, - )), - }; - - let serialized = serde_json::to_vec(&envelope).unwrap(); - let deserialized: RequestEnvelope = - serde_json::from_slice(serialized.as_slice()).unwrap(); - - assert!(envelope == deserialized, "serialization round trip failed"); - } -} diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index 708666cff94..2de32d0d7a0 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -6,6 +6,7 @@ use crate::bootstrap::rss_handle::run_rss; use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use bootstrap_agent_lockstep_types::MultirackJoinServiceState; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::RssStep; use dropshot::HttpError; @@ -14,7 +15,6 @@ use omicron_uuid_kinds::RackInitUuid; use sled_agent_bootstrap_common::RssContext; use sled_agent_multirack_join::MultirackJoinServiceError; use sled_agent_multirack_join::MultirackJoinServiceHandle; -use sled_agent_multirack_join::MultirackJoinServiceState; use sled_agent_rack_setup::RackInitializeRequestParams; use sled_agent_rack_setup::SetupServiceError; use slog_error_chain::InlineErrorChain; diff --git a/sled-agent/src/bootstrap/rss_handle.rs b/sled-agent/src/bootstrap/rss_handle.rs index dd7492eb370..e4c9d30d9d3 100644 --- a/sled-agent/src/bootstrap/rss_handle.rs +++ b/sled-agent/src/bootstrap/rss_handle.rs @@ -4,8 +4,6 @@ //! sled-agent's handle to the Rack Setup Service it spawns -use super::sprockets_client::SprocketsClient; -use super::sprockets_client::SprocketsClientError; use bootstrap_agent_lockstep_types::RssStep; use futures::StreamExt; use futures::stream::FuturesUnordered; @@ -13,6 +11,9 @@ use omicron_common::backoff::BackoffError; use omicron_common::backoff::retry_notify; use omicron_common::backoff::retry_policy_local; use sled_agent_bootstrap_common::RssContext; +use sled_agent_bootstrap_common::sprockets::{ + SprocketsClient, SprocketsClientError, +}; use sled_agent_measurements::MeasurementsHandle; use sled_agent_rack_setup::LocalBootstrapAgent; use sled_agent_rack_setup::RackInitializeRequestParams; diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index a7891fb0527..73808752483 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -7,7 +7,6 @@ use super::RssAccessError; use super::http_entrypoints_lockstep; use super::http_entrypoints_lockstep::BootstrapServerContext; -use super::views::SledAgentResponse; use crate::bootstrap::maghemite; use crate::bootstrap::pre_server::BootstrapAgentStartup; use crate::bootstrap::pumpkind; @@ -36,6 +35,7 @@ use omicron_ledger as ledger; use omicron_ledger::Ledger; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::RackInitUuid; +use sled_agent_bootstrap_common::sprockets::SledAgentResponse; use sled_agent_config_reconciler::ConfigReconcilerSpawnToken; use sled_agent_config_reconciler::InternalDisksReceiver; use sled_agent_rack_setup::RackInitializeRequestParams; diff --git a/sled-agent/src/bootstrap/sprockets_server.rs b/sled-agent/src/bootstrap/sprockets_server.rs index 00c3552ee7f..4854349b36e 100644 --- a/sled-agent/src/bootstrap/sprockets_server.rs +++ b/sled-agent/src/bootstrap/sprockets_server.rs @@ -4,12 +4,10 @@ //! Server for sprockets-secured requests over the bootstrap network. -use crate::bootstrap::params::Request; -use crate::bootstrap::params::RequestEnvelope; -use crate::bootstrap::params::version; -use crate::bootstrap::views::Response; -use crate::bootstrap::views::ResponseEnvelope; -use crate::bootstrap::views::SledAgentResponse; +use sled_agent_bootstrap_common::sprockets::{ + Request, RequestEnvelope, Response, ResponseEnvelope, SledAgentResponse, + version, +}; use sled_agent_measurements::MeasurementsHandle; use sled_agent_types::sled::StartSledAgentRequest; use slog::Logger; diff --git a/sled-agent/src/bootstrap/views.rs b/sled-agent/src/bootstrap/views.rs deleted file mode 100644 index dfc3a6ef321..00000000000 --- a/sled-agent/src/bootstrap/views.rs +++ /dev/null @@ -1,27 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Response types for the bootstrap agent - -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -/// Describes the Sled Agent running on the device. -#[derive(Serialize, Deserialize, PartialEq)] -pub struct SledAgentResponse { - pub id: Uuid, -} - -#[derive(Serialize, Deserialize, PartialEq)] -// Note: We intentionally do not derive `Debug` on this type, to avoid -// accidentally debug-logging the secret share. -pub enum Response { - SledAgentResponse(SledAgentResponse), -} - -#[derive(Serialize, Deserialize, PartialEq)] -pub struct ResponseEnvelope { - pub version: u32, - pub response: Result, -} diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b15ecbbf868..bf73df19d67 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -5,9 +5,6 @@ //! Sled agent implementation use crate::artifact_store::{ArtifactStore, SledAgentArtifactStoreWrapper}; -use crate::bootstrap::sprockets_client::{ - SprocketsClient, SprocketsClientError, -}; use crate::config::Config; use crate::hardware_monitor::HardwareMonitorHandle; use crate::instance_manager::InstanceManager; @@ -64,6 +61,9 @@ use omicron_uuid_kinds::{ }; use oximeter_instruments::http::LatencyTracker; use oxnet::IpNet; +use sled_agent_bootstrap_common::sprockets::{ + SprocketsClient, SprocketsClientError, +}; use sled_agent_config_reconciler::{ ConfigReconcilerHandle, ConfigReconcilerSpawnToken, InternalDisks, InternalDisksReceiver, LedgerNewConfigError, LedgerTaskError,