diff --git a/Cargo.lock b/Cargo.lock index c3b09ce691c..98ac5b8a015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -925,6 +925,7 @@ dependencies = [ "dropshot", "omicron-uuid-kinds", "omicron-workspace-hack", + "sled-agent-multirack-join", ] [[package]] @@ -960,10 +961,12 @@ dependencies = [ "omicron-workspace-hack", "schemars 0.8.22", "serde", + "serde_json", "sled-agent-types", "sled-hardware-types", "strum 0.27.2", "thiserror 2.0.18", + "trust-quorum-types", ] [[package]] @@ -9703,11 +9706,13 @@ dependencies = [ "sha3 0.10.8", "signal-hook", "sled-agent-api", + "sled-agent-bootstrap-common", "sled-agent-client", "sled-agent-config-reconciler", "sled-agent-early-networking", "sled-agent-health-monitor", "sled-agent-measurements", + "sled-agent-multirack-join", "sled-agent-rack-setup", "sled-agent-resolvable-files", "sled-agent-types", @@ -13997,6 +14002,24 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "sled-agent-bootstrap-common" +version = "0.1.0" +dependencies = [ + "bootstore", + "camino", + "omicron-ledger", + "omicron-workspace-hack", + "serde", + "sled-agent-config-reconciler", + "sled-agent-measurements", + "slog", + "slog-error-chain", + "sprockets-tls", + "thiserror 2.0.18", + "trust-quorum", +] + [[package]] name = "sled-agent-client" version = "0.1.0" @@ -14149,6 +14172,41 @@ dependencies = [ "tokio", ] +[[package]] +name = "sled-agent-multirack-join" +version = "0.1.0" +dependencies = [ + "bootstore", + "bootstrap-agent-lockstep-types", + "camino", + "camino-tempfile", + "itertools 0.14.0", + "nexus-types", + "omicron-common", + "omicron-ddm-admin-client", + "omicron-ledger", + "omicron-test-utils", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "oxnet", + "schemars 0.8.22", + "serde", + "sled-agent-bootstrap-common", + "sled-agent-types", + "sled-hardware-types", + "sled-storage", + "slog", + "slog-error-chain", + "static_assertions", + "thiserror 2.0.18", + "tokio", + "toml 0.8.23", + "trust-quorum", + "trust-quorum-protocol", + "trust-quorum-types", + "uuid", +] + [[package]] name = "sled-agent-rack-setup" version = "0.1.0" @@ -14185,6 +14243,7 @@ dependencies = [ "reqwest 0.13.2", "schemars 0.8.22", "serde", + "sled-agent-bootstrap-common", "sled-agent-client", "sled-agent-config-reconciler", "sled-agent-types", diff --git a/Cargo.toml b/Cargo.toml index b05c6670915..bb169b86ae2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -150,10 +150,12 @@ members = [ "sled-agent/api", "sled-agent/bootstrap-agent-lockstep-api", "sled-agent/bootstrap-agent-lockstep-types", + "sled-agent/bootstrap-common", "sled-agent/config-reconciler", "sled-agent/early-networking", "sled-agent/health-monitor", "sled-agent/measurements", + "sled-agent/multirack-join", "sled-agent/rack-setup", "sled-agent/repo-depot-api", "sled-agent/scrimlet-reconcilers", @@ -345,10 +347,12 @@ default-members = [ "sled-agent/api", "sled-agent/bootstrap-agent-lockstep-api", "sled-agent/bootstrap-agent-lockstep-types", + "sled-agent/bootstrap-common", "sled-agent/config-reconciler", "sled-agent/early-networking", "sled-agent/health-monitor", "sled-agent/measurements", + "sled-agent/multirack-join", "sled-agent/rack-setup", "sled-agent/repo-depot-api", "sled-agent/scrimlet-reconcilers", @@ -812,11 +816,13 @@ similar-asserts = "1.7.0" # server zones. sled = "=0.34.7" sled-agent-api = { path = "sled-agent/api" } +sled-agent-bootstrap-common = { path = "sled-agent/bootstrap-common" } sled-agent-client = { path = "clients/sled-agent-client" } sled-agent-config-reconciler = { path = "sled-agent/config-reconciler" } sled-agent-early-networking = { path = "sled-agent/early-networking" } sled-agent-health-monitor = { path = "sled-agent/health-monitor" } sled-agent-measurements = { path = "sled-agent/measurements" } +sled-agent-multirack-join = { path = "sled-agent/multirack-join" } sled-agent-rack-setup = { path = "sled-agent/rack-setup" } sled-agent-scrimlet-reconcilers = { path = "sled-agent/scrimlet-reconcilers" } sled-agent-types = { path = "sled-agent/types" } diff --git a/openapi/bootstrap-agent-lockstep.json b/openapi/bootstrap-agent-lockstep.json index 356b44ae07f..28bd3c03799 100644 --- a/openapi/bootstrap-agent-lockstep.json +++ b/openapi/bootstrap-agent-lockstep.json @@ -59,9 +59,65 @@ } } }, + "/multirack-join": { + "get": { + "summary": "Get the current state of the multirack join", + "operationId": "multirack_join_state", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MultirackJoinServiceState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "summary": "Configure the rack to join a multirack regional cluster", + "operationId": "multirack_join", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MultirackJoinRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MultirackJoinUuid" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/rack-initialize": { "get": { - "summary": "Get the current status of rack initialization or reset.", + "summary": "Get the current status of rack initialization.", "operationId": "rack_initialization_status", "responses": { "200": { @@ -513,6 +569,106 @@ "key" ] }, + "Configuration": { + "description": "The configuration for a given epoch.\n\nOnly valid for non-lrtq configurations.", + "type": "object", + "properties": { + "coordinator": { + "description": "Who was the coordinator of this reconfiguration?", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "encrypted_rack_secrets": { + "nullable": true, + "description": "There are no encrypted rack secrets for the initial configuration.", + "allOf": [ + { + "$ref": "#/components/schemas/EncryptedRackSecrets" + } + ] + }, + "epoch": { + "description": "Unique, monotonically increasing identifier for a configuration.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "members": { + "description": "All members of the current configuration and the hash of their key shares.", + "type": "array", + "items": { + "$ref": "#/components/schemas/ConfigurationMember" + } + }, + "rack_id": { + "description": "Unique Id of the rack.", + "allOf": [ + { + "$ref": "#/components/schemas/RackUuid" + } + ] + }, + "threshold": { + "description": "The number of sleds required to reconstruct the rack secret.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "coordinator", + "epoch", + "members", + "rack_id", + "threshold" + ] + }, + "ConfigurationMember": { + "description": "A member entry in a trust quorum configuration.\n\nThis type is used for OpenAPI schema generation since OpenAPI v3.0.x doesn't support tuple arrays.", + "type": "object", + "properties": { + "id": { + "description": "The baseboard ID of the member.", + "allOf": [ + { + "$ref": "#/components/schemas/BaseboardId" + } + ] + }, + "share_digest": { + "description": "The SHA3-256 hash of the member's key share.", + "type": "string", + "format": "hex string (32 bytes)" + } + }, + "required": [ + "id", + "share_digest" + ] + }, + "EncryptedRackSecrets": { + "description": "All possibly relevant __encrypted__ rack secrets for _prior_ committed configurations.", + "type": "object", + "properties": { + "data": { + "description": "Encrypted data.", + "type": "string", + "format": "hex string" + }, + "salt": { + "description": "A random value used to derive the key to encrypt the rack secrets for prior committed epochs.", + "type": "string", + "format": "hex string (32 bytes)" + } + }, + "required": [ + "data", + "salt" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -839,6 +995,278 @@ "minimum": 1, "maximum": 32 }, + "MultirackJoinRequest": { + "type": "object", + "properties": { + "rack_network_config": { + "description": "The rack network configuration for this joining rack", + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfig" + } + ] + }, + "trust_quorum_peers": { + "description": "The set of peers required to initialize trust quorum\n\nUnlike RSS, this is not optional for multirack setups. Bootstrap addresses are discovered by the bootstrap agent and mapped to the `BaseboardId`s.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + } + }, + "required": [ + "rack_network_config", + "trust_quorum_peers" + ] + }, + "MultirackJoinServiceState": { + "description": "The current state of the `MultirackJoinService` as retrieved from the `output_rx` watch channel.", + "oneOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "uninitialized" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "requested" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "starting" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "A request from Nexus informing a node to start coordinating a reconfiguration.", + "type": "object", + "properties": { + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_committed_epoch": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "members": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "state": { + "type": "string", + "enum": [ + "trust_quorum_reconfigure" + ] + }, + "threshold": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "epoch", + "members", + "rack_id", + "state", + "threshold" + ] + }, + { + "description": "Status of the node coordinating the reconfiguration or LRTQ upgrade.", + "type": "object", + "properties": { + "acked_prepares": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "config": { + "$ref": "#/components/schemas/Configuration" + }, + "state": { + "type": "string", + "enum": [ + "trust_quorum_preparing" + ] + } + }, + "required": [ + "acked_prepares", + "config", + "state" + ] + }, + { + "description": "The state of the commit phase of the trust quorum protocol", + "type": "object", + "properties": { + "acked": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "commit_crash_tolerance": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "epoch": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "fatal_errors": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "last_committed_epoch": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "members": { + "type": "array", + "items": { + "$ref": "#/components/schemas/BaseboardId" + }, + "uniqueItems": true + }, + "rack_id": { + "$ref": "#/components/schemas/RackUuid" + }, + "state": { + "type": "string", + "enum": [ + "trust_quorum_committing" + ] + }, + "threshold": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "transient_errors": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "acked", + "commit_crash_tolerance", + "epoch", + "fatal_errors", + "members", + "rack_id", + "state", + "threshold", + "transient_errors" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "completed" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "state": { + "type": "string", + "enum": [ + "failed" + ] + } + }, + "required": [ + "message", + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "task_panicked" + ] + } + }, + "required": [ + "state" + ] + } + ] + }, + "MultirackJoinUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::MultirackJoinUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID, but they may contain a UUID. They can be at most 63 characters long.", @@ -1115,6 +1543,20 @@ "RackOperationStatus": { "description": "Current status of any rack-level operation being performed by this bootstrap agent.", "oneOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "uninitialized" + ] + } + }, + "required": [ + "status" + ] + }, { "type": "object", "properties": { @@ -1203,19 +1645,94 @@ { "type": "object", "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, "status": { "type": "string", "enum": [ - "uninitialized" + "multirack_join_in_progress" + ] + } + }, + "required": [ + "id", + "status" + ] + }, + { + "type": "object", + "properties": { + "id": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/MultirackJoinUuid" + } + ] + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_completed" ] } }, "required": [ "status" ] + }, + { + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_failed" + ] + } + }, + "required": [ + "id", + "message", + "status" + ] + }, + { + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_panicked" + ] + } + }, + "required": [ + "id", + "status" + ] } ] }, + "RackUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::RackUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, "RecoverySiloConfig": { "description": "Configuration for the recovery silo created during rack setup.", "type": "object", diff --git a/openapi/wicketd.json b/openapi/wicketd.json index ad775859a3a..2a788565d8d 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -2762,6 +2762,15 @@ "rack_network_config" ] }, + "MultirackJoinUuid": { + "x-rust-type": { + "crate": "omicron-uuid-kinds", + "path": "omicron_uuid_kinds::MultirackJoinUuid", + "version": "*" + }, + "type": "string", + "format": "uuid" + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID, but they may contain a UUID. They can be at most 63 characters long.", @@ -3731,6 +3740,20 @@ "RackOperationStatus": { "description": "Current status of any rack-level operation being performed by this bootstrap agent.", "oneOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "uninitialized" + ] + } + }, + "required": [ + "status" + ] + }, { "type": "object", "properties": { @@ -3819,14 +3842,80 @@ { "type": "object", "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, "status": { "type": "string", "enum": [ - "uninitialized" + "multirack_join_in_progress" ] } }, "required": [ + "id", + "status" + ] + }, + { + "type": "object", + "properties": { + "id": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/MultirackJoinUuid" + } + ] + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_completed" + ] + } + }, + "required": [ + "status" + ] + }, + { + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_failed" + ] + } + }, + "required": [ + "id", + "message", + "status" + ] + }, + { + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/MultirackJoinUuid" + }, + "status": { + "type": "string", + "enum": [ + "multirack_join_panicked" + ] + } + }, + "required": [ + "id", "status" ] } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 68bfd63ea2a..75d5ac56a98 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -86,11 +86,13 @@ sha2.workspace = true sha3.workspace = true signal-hook.workspace = true sled-agent-api.workspace = true +sled-agent-bootstrap-common.workspace = true sled-agent-client.workspace = true sled-agent-config-reconciler.workspace = true sled-agent-early-networking.workspace = true sled-agent-health-monitor.workspace = true sled-agent-measurements.workspace = true +sled-agent-multirack-join.workspace = true sled-agent-rack-setup.workspace = true sled-agent-types.workspace = true sled-agent-types-versions.workspace = true diff --git a/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml b/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml index 5ea1332aa47..f188cf69f63 100644 --- a/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml +++ b/sled-agent/bootstrap-agent-lockstep-api/Cargo.toml @@ -11,5 +11,5 @@ workspace = true dropshot.workspace = true bootstrap-agent-lockstep-types.workspace = true omicron-uuid-kinds.workspace = true - omicron-workspace-hack.workspace = true +sled-agent-multirack-join.workspace = true diff --git a/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs index bd6e0d44ef5..1bd2982244e 100644 --- a/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-api/src/lib.rs @@ -10,17 +10,20 @@ //! it's expected that software components are on the same version. use bootstrap_agent_lockstep_types::BaseboardIds; +use bootstrap_agent_lockstep_types::MultirackJoinRequest; use bootstrap_agent_lockstep_types::RackInitializeRequest; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig; use dropshot::{HttpError, HttpResponseOk, RequestContext, TypedBody}; +use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; +use sled_agent_multirack_join::MultirackJoinServiceState; #[dropshot::api_description] pub trait BootstrapAgentLockstepApi { type Context; - /// Get the current status of rack initialization or reset. + /// Get the current status of rack initialization. #[endpoint { method = GET, path = "/rack-initialize", @@ -62,4 +65,23 @@ pub trait BootstrapAgentLockstepApi { async fn baseboard_ids( rqctx: RequestContext, ) -> Result, HttpError>; + + /// Configure the rack to join a multirack regional cluster + #[endpoint { + method = POST, + path = "/multirack-join", + }] + async fn multirack_join( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; + + /// Get the current state of the multirack join + #[endpoint { + method = GET, + path = "/multirack-join", + }] + async fn multirack_join_state( + rqctx: RequestContext, + ) -> Result, HttpError>; } diff --git a/sled-agent/bootstrap-agent-lockstep-types/Cargo.toml b/sled-agent/bootstrap-agent-lockstep-types/Cargo.toml index 3a59c841448..1bff67f72b0 100644 --- a/sled-agent/bootstrap-agent-lockstep-types/Cargo.toml +++ b/sled-agent/bootstrap-agent-lockstep-types/Cargo.toml @@ -19,5 +19,9 @@ sled-agent-types.workspace = true sled-hardware-types.workspace = true strum.workspace = true thiserror.workspace = true +trust-quorum-types.workspace = true omicron-workspace-hack.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/sled-agent/bootstrap-agent-lockstep-types/examples/multirack_join_request.rs b/sled-agent/bootstrap-agent-lockstep-types/examples/multirack_join_request.rs new file mode 100644 index 00000000000..a7dbfe1a60a --- /dev/null +++ b/sled-agent/bootstrap-agent-lockstep-types/examples/multirack_join_request.rs @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! One-off CLI: prints a hardcoded `MultirackJoinRequest` as JSON. + +use std::collections::BTreeSet; + +use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use sled_agent_types::early_networking::{ + BgpConfig, BgpPeerConfig, ImportExportPolicy, LinkFec, LinkSpeed, + LldpAdminStatus, LldpPortConfig, MaxPathConfig, PortConfig, + RackNetworkConfig, RouterPeerType, SwitchSlot, UplinkAddressConfig, + UplinkPorts, +}; +use sled_hardware_types::BaseboardId; + +fn main() { + let trust_quorum_peers: BTreeSet = [ + BaseboardId { + part_number: "913-0000019".to_owned(), + serial_number: "2FAKE003".to_owned(), + }, + BaseboardId { + part_number: "913-0000019".to_owned(), + serial_number: "2FAKE004".to_owned(), + }, + BaseboardId { + part_number: "913-0000019".to_owned(), + serial_number: "2FAKE005".to_owned(), + }, + ] + .into_iter() + .collect(); + + let rack_network_config = RackNetworkConfig { + rack_subnet: "fd00:1122:3344:200::/56".parse().unwrap(), + infra_ip_first: "172.20.16.21".parse().unwrap(), + infra_ip_last: "172.20.16.22".parse().unwrap(), + ports: UplinkPorts::new(vec![PortConfig { + routes: vec![], + addresses: vec![UplinkAddressConfig::without_vlan( + "172.20.16.45/29".parse().unwrap(), + )], + switch: SwitchSlot::Switch0, + port: "qsfp0".to_owned(), + uplink_port_speed: LinkSpeed::Speed100G, + uplink_port_fec: Some(LinkFec::Rs), + bgp_peers: vec![BgpPeerConfig { + asn: 65002, + port: "qsfp0".to_owned(), + addr: RouterPeerType::Numbered { + ip: "172.20.16.43".parse().unwrap(), + }, + hold_time: Some(6), + idle_hold_time: Some(0), + delay_open: Some(3), + connect_retry: Some(3), + keepalive: Some(2), + remote_asn: None, + min_ttl: None, + md5_auth_key: None, + multi_exit_discriminator: None, + communities: Vec::new(), + local_pref: None, + enforce_first_as: false, + allowed_import: ImportExportPolicy::NoFiltering, + allowed_export: ImportExportPolicy::NoFiltering, + vlan_id: None, + }], + autoneg: false, + tx_eq: None, + lldp: Some(LldpPortConfig { + status: LldpAdminStatus::Disabled, + chassis_id: None, + port_id: None, + port_description: None, + system_name: None, + system_description: None, + management_addrs: None, + }), + }]) + .expect("port list is non-empty"), + bgp: vec![BgpConfig { + asn: 65002, + originate: vec!["172.20.52.0/22".parse().unwrap()], + shaper: None, + checker: None, + max_paths: MaxPathConfig::default(), + }], + bfd: vec![], + }; + + let req = MultirackJoinRequest { trust_quorum_peers, rack_network_config }; + + println!("{}", serde_json::to_string_pretty(&req).unwrap()); +} diff --git a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs index 2ee5fd54430..f5c6e5a085b 100644 --- a/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs +++ b/sled-agent/bootstrap-agent-lockstep-types/src/lib.rs @@ -25,6 +25,7 @@ use omicron_common::api::external::Error; use omicron_common::api::external::Name; use omicron_common::api::external::UserId; use omicron_common::api::internal::nexus::Certificate; +use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -238,6 +239,19 @@ fn validate_external_dns( Ok(()) } +#[derive(Clone, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct MultirackJoinRequest { + /// The set of peers required to initialize trust quorum + /// + /// Unlike RSS, this is not optional for multirack setups. Bootstrap + /// addresses are discovered by the bootstrap agent and mapped to the + /// `BaseboardId`s. + pub trust_quorum_peers: BTreeSet, + + /// The rack network configuration for this joining rack + pub rack_network_config: RackNetworkConfig, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "snake_case", tag = "type")] pub enum BootstrapAddressDiscovery { @@ -262,6 +276,7 @@ pub struct RecoverySiloConfig { )] #[serde(tag = "status", rename_all = "snake_case")] pub enum RackOperationStatus { + Uninitialized, Initializing { id: RackInitUuid, step: RssStep, @@ -277,7 +292,20 @@ pub enum RackOperationStatus { InitializationPanicked { id: RackInitUuid, }, - Uninitialized, + MultirackJoinInProgress { + // Status is queried via a different API + id: MultirackJoinUuid, + }, + MultirackJoinCompleted { + id: Option, + }, + MultirackJoinFailed { + id: MultirackJoinUuid, + message: String, + }, + MultirackJoinPanicked { + id: MultirackJoinUuid, + }, } /// Steps we go through during initial rack setup. diff --git a/sled-agent/bootstrap-common/Cargo.toml b/sled-agent/bootstrap-common/Cargo.toml new file mode 100644 index 00000000000..9368fd209b1 --- /dev/null +++ b/sled-agent/bootstrap-common/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "sled-agent-bootstrap-common" +version = "0.1.0" +edition.workspace = true +license = "MPL-2.0" +description = "shared code for bootstrap agent related crates" + +[lints] +workspace = true + +[dependencies] +bootstore.workspace = true +camino.workspace = true +omicron-ledger.workspace = true +serde.workspace = true +sled-agent-config-reconciler.workspace = true +sled-agent-measurements.workspace = true +slog-error-chain.workspace = true +slog.workspace = true +sprockets-tls.workspace = true +thiserror.workspace = true +trust-quorum.workspace = true + +omicron-workspace-hack.workspace = true diff --git a/sled-agent/bootstrap-common/src/lib.rs b/sled-agent/bootstrap-common/src/lib.rs new file mode 100644 index 00000000000..d1d85b8734b --- /dev/null +++ b/sled-agent/bootstrap-common/src/lib.rs @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Types and functionality shared among bootstrap agent related crates +//! +//! This is unversioned, internal data. It's only expected to be used by RSS +//! and the Multirack Join Service. Please do not use it in public facing, +//! non-lockstep, interfaces. + +use bootstore::schemes::v0 as bootstore; +use camino::Utf8PathBuf; +use omicron_ledger::{Ledger, Ledgerable}; +use serde::{Deserialize, Serialize}; +use sled_agent_config_reconciler::InternalDisksReceiver; +use sled_agent_measurements::MeasurementsHandle; +use slog::{Logger, error, info}; +use slog_error_chain::SlogInlineError; +use sprockets_tls::keys::SprocketsConfig; +use std::net::Ipv6Addr; +use std::sync::Arc; +use thiserror::Error; + +#[derive(Clone, Serialize, Deserialize, Default)] +struct RssStartedMarker {} + +impl Ledgerable for RssStartedMarker { + fn is_newer_than(&self, _other: &Self) -> bool { + true + } + fn generation_bump(&mut self) {} +} + +const RSS_STARTED_FILENAME: &str = "rss-started.marker"; + +#[derive(Clone, Serialize, Deserialize, Default)] +struct RssCompleteMarker {} + +impl Ledgerable for RssCompleteMarker { + fn is_newer_than(&self, _other: &Self) -> bool { + true + } + fn generation_bump(&mut self) {} +} +const RSS_COMPLETED_FILENAME: &str = "rss-plan-completed.marker"; + +/// Functionality necessary to run rack initialization or multirack cluster join +/// +/// This is usually input to some constructor function +#[derive(Clone)] +pub struct RssContext { + pub base_log: Logger, + pub global_zone_bootstrap_ip: Ipv6Addr, + pub internal_disks_rx: InternalDisksReceiver, + pub bootstore_node_handle: bootstore::NodeHandle, + pub sprockets_config: SprocketsConfig, + pub trust_quorum_handle: trust_quorum::NodeTaskHandle, + pub measurements: Arc, +} + +impl RssContext { + fn started_marker_paths(&self) -> Vec { + let config_dataset_paths = self + .internal_disks_rx + .current() + .all_config_datasets() + .collect::>(); + + config_dataset_paths + .iter() + .map(|p| p.join(RSS_STARTED_FILENAME)) + .collect() + } + + fn completed_marker_paths(&self) -> Vec { + let config_dataset_paths = self + .internal_disks_rx + .current() + .all_config_datasets() + .collect::>(); + + config_dataset_paths + .iter() + .map(|p| p.join(RSS_COMPLETED_FILENAME)) + .collect() + } + + /// Check if a previous RSS or multirack join has completed successfully. + /// + /// If we see the completion marker in the `completed_ledger` then the + /// system should be up-and-running. If we see the started marker in + /// the `started_ledger`, then RSS or multirack join did not complete and + /// the rack should be clean-slated before either is run again. + pub async fn is_rss_complete( + &self, + log: &Logger, + ) -> Result<(), RunRssError> { + info!(log, "Checking to see if RSS configuration is complete"); + let started_ledger = + Ledger::::new(log, self.started_marker_paths()) + .await; + let completed_ledger = Ledger::::new( + log, + self.completed_marker_paths(), + ) + .await; + + if completed_ledger.is_some() { + info!(log, "RSS configuration has already been applied",); + return Err(RunRssError::RackAlreadyInitialized); + } else if started_ledger.is_some() { + error!(log, "RSS failed to complete rack initialization"); + return Err(RunRssError::RackInitInterrupted); + } + + Ok(()) + } + + pub async fn write_rss_started_ledger( + &self, + log: &Logger, + ) -> Result<(), omicron_ledger::Error> { + // Record that we have started RSS + let mut ledger = Ledger::::new_with( + log, + self.started_marker_paths(), + RssStartedMarker::default(), + ); + ledger.commit().await + } + + pub async fn write_rss_completed_ledger( + &self, + log: &Logger, + ) -> Result<(), omicron_ledger::Error> { + let mut ledger = Ledger::::new_with( + log, + self.completed_marker_paths(), + RssCompleteMarker::default(), + ); + ledger.commit().await + } +} + +#[derive(Error, Debug, SlogInlineError)] +pub enum RunRssError { + #[error("Rack already initialized")] + RackAlreadyInitialized, + + #[error("Rack initialization was interrupted. Clean-slate required")] + RackInitInterrupted, +} diff --git a/sled-agent/multirack-join/Cargo.toml b/sled-agent/multirack-join/Cargo.toml new file mode 100644 index 00000000000..421a6a84a0c --- /dev/null +++ b/sled-agent/multirack-join/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "sled-agent-multirack-join" +version = "0.1.0" +edition.workspace = true +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +bootstore.workspace = true +bootstrap-agent-lockstep-types.workspace = true +camino.workspace = true +itertools.workspace = true +nexus-types.workspace = true +omicron-common.workspace = true +omicron-ddm-admin-client.workspace = true +omicron-ledger.workspace = true +omicron-uuid-kinds.workspace = true +omicron-workspace-hack.workspace = true +oxnet.workspace = true +schemars.workspace = true +serde.workspace = true +sled-agent-types.workspace = true +sled-agent-bootstrap-common.workspace = true +sled-hardware-types.workspace = true +sled-storage.workspace = true +slog.workspace = true +slog-error-chain.workspace = true +static_assertions.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["full"] } +toml.workspace = true +trust-quorum.workspace = true +trust-quorum-protocol.workspace = true +trust-quorum-types.workspace = true +uuid.workspace = true + +[dev-dependencies] +camino-tempfile.workspace = true +omicron-test-utils.workspace = true diff --git a/sled-agent/multirack-join/src/lib.rs b/sled-agent/multirack-join/src/lib.rs new file mode 100644 index 00000000000..275fdced715 --- /dev/null +++ b/sled-agent/multirack-join/src/lib.rs @@ -0,0 +1,590 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Multirack Join Service +//! +//! This is the bootstrap service that provisions a new rack on an existing +//! underlay network such that it can be adopted by an existing Nexus in a +//! cluster on the same network. It's vastly simpler than RSS because only +//! sled-agent is started. No other control plane zones, including DNS and NTP, +//! are started. Reconfigurator on the existing Nexuses will setup the rack post +//! cluster join. +//! +//! See RFD 680 for further details. + +#[macro_use] +extern crate slog; +use bootstrap_agent_lockstep_types::MultirackJoinRequest; +use nexus_types::trust_quorum::TrustQuorumConfig; +use omicron_uuid_kinds::RackUuid; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sled_agent_bootstrap_common::{RssContext, RunRssError}; +use sled_hardware_types::BaseboardId; +use slog::{Logger, info}; +use slog_error_chain::{InlineErrorChain, SlogInlineError}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, Mutex}, + time::Duration, +}; +use thiserror::Error; +use tokio::{ + sync::watch, + task::{JoinError, JoinSet}, +}; +use trust_quorum::{NodeApiError, ProxyError}; +use trust_quorum_types::{ + messages::ReconfigureMsg as TqReconfigureMsg, + status::CoordinatorStatus, + types::{Epoch, Threshold}, +}; + +const INITIAL_TRUST_QUORUM_EPOCH: Epoch = Epoch(1); +const TRUST_QUORUM_RETRY_TIMEOUT: Duration = Duration::from_millis(500); + +/// Describes errors which may occur while operating the multirack join service. +#[derive(Error, Debug, SlogInlineError)] +pub enum MultirackJoinServiceError { + #[error("Rack already initialized")] + RackAlreadyInitialized, + + #[error("Rack initialization was interrupted. Clean-slate required")] + RackInitInterrupted, + + #[error("Trust quorum error")] + TqApiError(#[from] NodeApiError), + + #[error("Trust quorum coordinator doesn't think it's a coordinator")] + TqBadCoordinator, + + #[error("Trust quorum commit cannot complete: {0:?}")] + TqCommitFailed(CommitState), + + #[error("Failed to receive input from bootstrap agent")] + InputRx(#[from] watch::error::RecvError), + + #[error("Failed to join proxy commit task")] + ProxyCommit(#[from] JoinError), +} + +impl From for MultirackJoinServiceError { + fn from(value: RunRssError) -> Self { + match value { + RunRssError::RackAlreadyInitialized => Self::RackAlreadyInitialized, + RunRssError::RackInitInterrupted => Self::RackInitInterrupted, + } + } +} + +/// The state of the commit phase of the trust quorum protocol +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +pub struct CommitState { + rack_id: RackUuid, + members: BTreeSet, + epoch: Epoch, + last_committed_epoch: Option, + threshold: Threshold, + commit_crash_tolerance: u8, + acked: BTreeSet, + fatal_errors: BTreeMap, + transient_errors: BTreeMap, +} + +/// The current state of the `MultirackJoinService` as retrieved from the +/// `output_rx` watch channel. +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "snake_case", tag = "state")] +pub enum MultirackJoinServiceState { + Uninitialized, + Requested, + Starting, + TrustQuorumReconfigure(TqReconfigureMsg), + TrustQuorumPreparing(CoordinatorStatus), + TrustQuorumCommitting(CommitState), + Completed, + Failed { message: String }, + TaskPanicked, +} + +// The value returned from `MultirackJoinServiceTask::tq_prepare` +enum TqPrepareResult { + Prepared, + ReconfigurationNeeded { + new_members: BTreeSet, + new_epoch: Epoch, + }, +} + +// The value returned from `MultirackJoinServiceTask::tq_commit` +enum TqCommitResult { + Committed, + ReconfigurationNeeded { + new_members: BTreeSet, + new_epoch: Epoch, + just_committed_epoch: Epoch, + }, +} + +/// The interface to the Multirack Join Service. +pub struct MultirackJoinServiceHandle { + pub join_handle: + tokio::task::JoinHandle>, + pub input_tx: watch::Sender, + pub output_rx: watch::Receiver, +} + +impl MultirackJoinServiceHandle { + pub fn spawn(ctx: RssContext, request: MultirackJoinRequest) -> Self { + let (input_tx, input_rx) = watch::channel(request); + let state = MultirackJoinServiceState::Requested; + let (output_tx, output_rx) = watch::channel(state.clone()); + let join_handle = tokio::task::spawn(async move { + let log = + ctx.base_log.new(o!("component" => "MultirackJoinService")); + info!(log, "Starting Multirack Join Service"); + let mut task = + MultirackJoinServiceTask { log, ctx, input_rx, output_tx }; + task.run().await + }); + + Self { join_handle, input_tx, output_rx } + } +} + +/// The internal state of the main task running the join service +struct MultirackJoinServiceTask { + log: Logger, + ctx: RssContext, + input_rx: watch::Receiver, + output_tx: watch::Sender, +} + +impl MultirackJoinServiceTask { + /// The main loop of the Multirack Join Service + pub async fn run(&mut self) -> Result<(), MultirackJoinServiceError> { + self.output_tx + .send_modify(|state| *state = MultirackJoinServiceState::Starting); + + // Check to see if we've already finished RSS or multirack join + self.ctx.is_rss_complete(&self.log).await?; + info!(&self.log, "No RSS ledger found. Starting Multirack Join Setup"); + + let rack_id = RackUuid::new_v4(); + info!(&self.log, "Created RackId {rack_id}"); + + self.init_trust_quorum(rack_id).await?; + + // TODO: + // Start sled-agents + // Configure networking + // + // https://github.com/oxidecomputer/omicron/issues/10637 + + Ok(()) + } + + /// Start initializing trust quorum given the the existing + /// `MultirackJoinRequest` in input_rx. + /// + /// If we get an update on `input_rx` while trust quorum is stuck + /// waiting for all nodes to prepare or commit, we will skip the existing + /// configuration, or commit it and move on. If too many nodes are offline + /// during commit, we will not be able to make progress. But that is true in + /// general. Automatic cleanup is generally only valid for a few offline / + /// misbehaving nodes once commit phase is entered. + async fn init_trust_quorum( + &mut self, + rack_id: RackUuid, + ) -> Result<(), MultirackJoinServiceError> { + let members = + self.input_rx.borrow_and_update().trust_quorum_peers.clone(); + let epoch = INITIAL_TRUST_QUORUM_EPOCH; + let last_committed_epoch = None; + + self.tq_run(rack_id, members, epoch, last_committed_epoch).await + } + + /// Start the reconfigure/prepare/commit process with the given values + async fn tq_run( + &mut self, + rack_id: RackUuid, + mut members: BTreeSet, + mut epoch: Epoch, + mut last_committed_epoch: Option, + ) -> Result<(), MultirackJoinServiceError> { + loop { + self.tq_reconfigure( + rack_id, + members.clone(), + epoch, + last_committed_epoch, + ) + .await?; + + if let TqPrepareResult::ReconfigurationNeeded { + new_members, + new_epoch, + } = self.tq_prepare(members.clone(), epoch).await? + { + // Start over + members = new_members; + epoch = new_epoch; + continue; + }; + + match self + .tq_commit(rack_id, members, epoch, last_committed_epoch) + .await? + { + TqCommitResult::Committed => { + // We are done. + break; + } + TqCommitResult::ReconfigurationNeeded { + new_members, + new_epoch, + just_committed_epoch, + } => { + members = new_members; + epoch = new_epoch; + last_committed_epoch = Some(just_committed_epoch); + } + } + } + + Ok(()) + } + + async fn tq_reconfigure( + &mut self, + rack_id: RackUuid, + members: BTreeSet, + epoch: Epoch, + last_committed_epoch: Option, + ) -> Result<(), MultirackJoinServiceError> { + let threshold = + TrustQuorumConfig::threshold(members.len().try_into().unwrap()); + + info!( + self.log, + "Starting trust quorum reconfiguration"; + "epoch" => %epoch, + "last_committed_epoch" => ?last_committed_epoch, + "threshold" => %threshold + ); + + let msg = TqReconfigureMsg { + rack_id, + epoch, + last_committed_epoch, + members, + threshold, + }; + + self.output_tx.send_modify(|state| { + *state = + MultirackJoinServiceState::TrustQuorumReconfigure(msg.clone()) + }); + + // Start the initial configuration with this node as coordinator + self.ctx.trust_quorum_handle.reconfigure(msg).await?; + + info!( + self.log, + "Trust quorum reconfiguration started"; + "epoch" => %epoch, + "last_committed_epoch" => ?last_committed_epoch, + "threshold" => %threshold + ); + + Ok(()) + } + + async fn tq_prepare( + &mut self, + members: BTreeSet, + epoch: Epoch, + ) -> Result { + loop { + let status = self + .ctx + .trust_quorum_handle + .coordinator_status() + .await? + .ok_or(MultirackJoinServiceError::TqBadCoordinator)?; + + let all_nodes_prepared = status.acked_prepares == members; + let still_waiting = itertools::join( + members.difference(&status.acked_prepares), + ",", + ); + + // Set the output state and notifiy receivers if it has changed + self.output_tx.send_if_modified(|state| { + let new_state = + MultirackJoinServiceState::TrustQuorumPreparing(status); + if *state == new_state { + false + } else { + *state = new_state; + true + } + }); + + // We're done preparing. Let's move on to committing. + if all_nodes_prepared { + info!( + self.log, + "Trust quorum prepared at all nodes"; + "epoch" => %epoch + ); + break; + } + + info!( + self.log, + "trust quorum coordinator waiting for PrepareAcks"; + "epoch" => %epoch, + "waiting_for" => still_waiting + ); + + // Before we check our prepare status again let's see if we've + // received an updated configuration from an operator. + // + // The prepare phase of the TQ protocol can be interrupted safely at + // any time, even if all nodes have received the `Prepare` message. + if let Some(new_members) = + self.has_membership_changed(&members).await? + { + return Ok(TqPrepareResult::ReconfigurationNeeded { + new_members, + new_epoch: epoch.next(), + }); + } + + tokio::time::sleep(TRUST_QUORUM_RETRY_TIMEOUT).await; + } + + // Before we return, let's see if the operator has changed the + // configuration one more time. + if let Some(new_members) = self.has_membership_changed(&members).await? + { + Ok(TqPrepareResult::ReconfigurationNeeded { + new_members, + new_epoch: epoch.next(), + }) + } else { + Ok(TqPrepareResult::Prepared) + } + } + + // Attempt to commit the configuration at each node. + // + // If a new configuration comes in before commit is attempted or after the + // minimum number of nodes has committed then return the new members, new + // epoch, and last commmitted epoch. + async fn tq_commit( + &mut self, + rack_id: RackUuid, + members: BTreeSet, + epoch: Epoch, + last_committed_epoch: Option, + ) -> Result { + info!( + self.log, + "Starting to commit trust quorum configuration"; + "epoch" => %epoch + ); + + // Commit at this node. This is the only node that we locally access + // over the bootstrap agent. For security purposes, we proxy all other + // requests over sprockets. + // + // Unfortunately, if we have a problem here, we need to stop what we're + // doing and clean slate. + self.ctx.trust_quorum_handle.commit(rack_id, epoch).await?; + + // Peers that must proxy commit over sprockets + let mut remote_peers = members.clone(); + let this_sled = self.ctx.trust_quorum_handle.baseboard_id().clone(); + remote_peers.remove(&this_sled); + let mut set = JoinSet::new(); + + let threshold = + TrustQuorumConfig::threshold(members.len().try_into().unwrap()); + let commit_crash_tolerance = TrustQuorumConfig::commit_crash_tolerance( + members.len().try_into().unwrap(), + ); + let min_acks_to_commit = + (threshold.0 + commit_crash_tolerance) as usize; + + // Transient errors are updated inside proxy commit tasks + let transient_errors = Arc::new(Mutex::new(BTreeMap::new())); + + // Update our state as we start + let mut commit_state = CommitState { + rack_id, + members: members.clone(), + epoch, + last_committed_epoch, + threshold, + commit_crash_tolerance, + acked: BTreeSet::from([this_sled]), + fatal_errors: BTreeMap::new(), + transient_errors: BTreeMap::new(), + }; + self.output_tx.send_modify(|state| { + *state = MultirackJoinServiceState::TrustQuorumCommitting( + commit_state.clone(), + ) + }); + + for peer in remote_peers { + self.tq_spawn_proxy_commit_task( + rack_id, + peer, + epoch, + transient_errors.clone(), + &mut set, + ); + } + + loop { + tokio::select! { + Some(res) = set.join_next() => { + match res? { + Ok(peer_id) => { + info!( + self.log, + "Proxy commit acked"; + "peer_id" => %peer_id + ); + commit_state.acked.insert(peer_id); + } + Err((peer_id,err)) => { + error!( + self.log, + "Failed to proxy commit"; + "peer_id" => %peer_id, + "err" => %err + ); + commit_state.fatal_errors.insert(peer_id, err); + + } + } + self.output_tx.send_modify(|state| { + *state = MultirackJoinServiceState::TrustQuorumCommitting( + commit_state.clone(), + ) + }); + if commit_state.acked == members { + info!( + self.log, + "Trust quorum committed at all nodes"; + "epoch" => %epoch + ); + break; + } + + } + res = self.input_rx.changed() => { + res?; + // The operator changed the input + let new_members = self + .input_rx + .borrow_and_update() + .trust_quorum_peers.clone(); + + // Did the membership change? + if members != new_members { + if commit_state.acked.len() >= min_acks_to_commit { + // We can commit this configuration safely and then + // try a reconfiguration with the updated config. + let just_committed_epoch = epoch; + return Ok(TqCommitResult::ReconfigurationNeeded { + new_members, + new_epoch: epoch.next(), + just_committed_epoch + }); + } + return Err(MultirackJoinServiceError::TqCommitFailed( + commit_state + )); + } else { + // Something other than membership changed. Ignore. + } + } + } + } + + Ok(TqCommitResult::Committed) + } + + /// Spawn a task to perform a proxy commit + /// + /// Success and fatal errors are returned from the task. Transient errors + /// are continuously retried. + fn tq_spawn_proxy_commit_task( + &mut self, + rack_id: RackUuid, + peer: BaseboardId, + epoch: Epoch, + transient_errors: Arc>>, + set: &mut JoinSet>, + ) { + let proxy = self.ctx.trust_quorum_handle.proxy(); + info!( + self.log, + "Attempting to proxy commit trust quorum"; + "epoch" => %epoch, + "baseboard_id" => %peer + ); + set.spawn(async move { + loop { + let peer = peer.clone(); + match proxy.commit(peer.clone(), rack_id, epoch).await { + Ok(trust_quorum_types::status::CommitStatus::Committed) => { + return Ok(peer); + } + Ok(trust_quorum_types::status::CommitStatus::Pending) => { + let s = "unexpected CommitStatus::Pending \ + from prepared peer" + .to_string(); + return Err((peer, s)); + } + Err(e @ ProxyError::Inner(_)) + | Err(e @ ProxyError::InvalidResponse(_)) + | Err(e @ ProxyError::RecvError) => { + let s = InlineErrorChain::new(&e).to_string(); + return Err((peer, s)); + } + Err(e) => { + let s = InlineErrorChain::new(&e).to_string(); + transient_errors.lock().unwrap().insert(peer, s); + tokio::time::sleep(TRUST_QUORUM_RETRY_TIMEOUT).await; + } + } + } + }); + } + + // Check if we have received an updated membership set from an operator. + // + // If we have received a new set, return it. Otherwise, return `None`. + // Return an error if checking for the update fails. + async fn has_membership_changed( + &mut self, + members: &BTreeSet, + ) -> Result>, MultirackJoinServiceError> { + if self.input_rx.has_changed()? { + let new_members = + self.input_rx.borrow_and_update().trust_quorum_peers.clone(); + if new_members != *members { + return Ok(Some(new_members)); + } + } + Ok(None) + } +} diff --git a/sled-agent/rack-setup/Cargo.toml b/sled-agent/rack-setup/Cargo.toml index e16019aee5a..a6aac366913 100644 --- a/sled-agent/rack-setup/Cargo.toml +++ b/sled-agent/rack-setup/Cargo.toml @@ -37,6 +37,7 @@ reqwest = { workspace = true, features = ["rustls", "stream"] } schemars = { workspace = true, features = ["chrono", "uuid1"] } serde.workspace = true sled-agent-client.workspace = true +sled-agent-bootstrap-common.workspace = true sled-agent-config-reconciler.workspace = true sled-agent-types.workspace = true sled-hardware-types.workspace = true diff --git a/sled-agent/rack-setup/src/service.rs b/sled-agent/rack-setup/src/service.rs index 3fc3c640568..c090b42d266 100644 --- a/sled-agent/rack-setup/src/service.rs +++ b/sled-agent/rack-setup/src/service.rs @@ -99,15 +99,15 @@ use omicron_common::backoff::{ }; use omicron_common::disk::DatasetKind; use omicron_ddm_admin_client::DdmError; -use omicron_ledger::{self as ledger, Ledger, Ledgerable}; +use omicron_ledger::{self as ledger}; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::RackUuid; use omicron_uuid_kinds::ZpoolUuid; -use serde::{Deserialize, Serialize}; +use sled_agent_bootstrap_common::RssContext; +use sled_agent_bootstrap_common::RunRssError; use sled_agent_client::{ Client as SledAgentClient, Error as SledAgentError, types as SledAgentTypes, }; -use sled_agent_config_reconciler::InternalDisksReceiver; use sled_agent_types::early_networking::EarlyNetworkConfigEnvelope; use sled_agent_types::inventory::{ ConfigReconcilerInventoryResult, HostPhase2DesiredSlots, OmicronSledConfig, @@ -124,7 +124,7 @@ use slog_error_chain::{InlineErrorChain, SlogInlineError}; use std::collections::BTreeSet; use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::net::{Ipv6Addr, SocketAddrV6}; +use std::net::SocketAddrV6; use std::time::Duration; use thiserror::Error; use tokio::sync::watch; @@ -263,6 +263,15 @@ pub enum SetupServiceError { TrustQuorumProxyCommitPending(BaseboardId), } +impl From for SetupServiceError { + fn from(value: RunRssError) -> Self { + match value { + RunRssError::RackAlreadyInitialized => Self::RackAlreadyInitialized, + RunRssError::RackInitInterrupted => Self::RackInitInterrupted, + } + } +} + #[derive(Debug, Clone)] pub struct RackInitializeRequestParams { pub rack_initialize_request: RackInitializeRequest, @@ -285,41 +294,17 @@ pub struct RackSetupService { impl RackSetupService { /// Creates a new rack setup service, which runs in a background task. - /// - /// Arguments: - /// - `log`: The logger. - /// - `config`: The config file, which is used to setup the rack. - /// - `internal_disks_rx`: Tells us about available internal disks - /// - `local_bootstrap_agent`: Communication channel by which we can send - /// commands to our local bootstrap-agent (e.g., to start sled-agents) - /// - `our_bootstrap_address`: The bootstrap address of the sled - /// hosting RSS (i.e., this sled). - /// - `bootstore` - A handle to call bootstore APIs - /// - `trust_quorum` - A handle to the trust qurom task - #[expect(clippy::too_many_arguments)] pub fn new( - log: Logger, + ctx: RssContext, request: RackInitializeRequestParams, - internal_disks_rx: InternalDisksReceiver, local_bootstrap_agent: T, - our_bootstrap_address: Ipv6Addr, - bootstore: bootstore::NodeHandle, - trust_quorum: trust_quorum::NodeTaskHandle, step_tx: watch::Sender, ) -> Self { let handle = tokio::task::spawn(async move { + let log = ctx.base_log.new(o!("component" => "RSS")); let svc = ServiceInner::new(log.clone()); - if let Err(e) = svc - .run( - &request, - &internal_disks_rx, - local_bootstrap_agent, - our_bootstrap_address, - bootstore, - trust_quorum, - step_tx, - ) - .await + if let Err(e) = + svc.run(ctx, &request, local_bootstrap_agent, step_tx).await { error!(log, "RSS injection failed"; &e); Err(e) @@ -337,29 +322,6 @@ impl RackSetupService { } } -#[derive(Clone, Serialize, Deserialize, Default)] -struct RssStartedMarker {} - -impl Ledgerable for RssStartedMarker { - fn is_newer_than(&self, _other: &Self) -> bool { - true - } - fn generation_bump(&mut self) {} -} - -const RSS_STARTED_FILENAME: &str = "rss-started.marker"; - -#[derive(Clone, Serialize, Deserialize, Default)] -struct RssCompleteMarker {} - -impl Ledgerable for RssCompleteMarker { - fn is_newer_than(&self, _other: &Self) -> bool { - true - } - fn generation_bump(&mut self) {} -} -const RSS_COMPLETED_FILENAME: &str = "rss-plan-completed.marker"; - /// The implementation of the Rack Setup Service. struct ServiceInner { log: Logger, @@ -1043,15 +1005,11 @@ impl ServiceInner { // rack, a marker file is created at "rss_completed_marker_path()". This // indicates that the plan executed successfully, and the only work // remaining is to handoff to Nexus. - #[expect(clippy::too_many_arguments)] async fn run( &self, + ctx: RssContext, request: &RackInitializeRequestParams, - internal_disks_rx: &InternalDisksReceiver, local_bootstrap_agent: T, - our_bootstrap_address: Ipv6Addr, - bootstore: bootstore::NodeHandle, - trust_quorum: trust_quorum::NodeTaskHandle, step_tx: watch::Sender, ) -> Result<(), SetupServiceError> { info!(self.log, "Injecting RSS configuration: {:#?}", request); @@ -1064,52 +1022,15 @@ impl ServiceInner { config.az_subnet(), )?; - let config_dataset_paths = internal_disks_rx - .current() - .all_config_datasets() - .collect::>(); - - let started_marker_paths: Vec = config_dataset_paths - .iter() - .map(|p| p.join(RSS_STARTED_FILENAME)) - .collect(); - - let completed_marker_paths: Vec = config_dataset_paths - .iter() - .map(|p| p.join(RSS_COMPLETED_FILENAME)) - .collect(); - - let started_ledger = Ledger::::new( - &self.log, - started_marker_paths.clone(), - ) - .await; - let completed_ledger = Ledger::::new( - &self.log, - completed_marker_paths.clone(), - ) - .await; - - // Check if a previous RSS plan has completed successfully. - // - // If we see the completion marker in the `completed_ledger` then the - // system should be up-and-running. If we see the started marker in - // the `started_ledger`, then RSS did not complete and the rack should - // be clean-slated before RSS is run again. - if completed_ledger.is_some() { - info!(self.log, "RSS configuration has already been applied",); - return Err(SetupServiceError::RackAlreadyInitialized); - } else if started_ledger.is_some() { - error!(self.log, "RSS failed to complete rack initialization"); - return Err(SetupServiceError::RackInitInterrupted); - } + // Check to see if we've already finished RSS or multirack join + ctx.is_rss_complete(&self.log).await?; info!(self.log, "RSS not previously run. Creating plans."); // Wait for enough peers to create a new plan let bootstrap_addrs = match &config.bootstrap_discovery { BootstrapAddressDiscovery::OnlyOurs => { - BTreeSet::from([our_bootstrap_address]) + BTreeSet::from([ctx.global_zone_bootstrap_ip]) } BootstrapAddressDiscovery::OnlyThese { addrs } => addrs.clone(), }; @@ -1128,12 +1049,7 @@ impl ServiceInner { // clean-slate and try again. // Record that we have started RSS - let mut ledger = Ledger::::new_with( - &self.log, - started_marker_paths.clone(), - RssStartedMarker::default(), - ); - ledger.commit().await?; + ctx.write_rss_started_ledger(&self.log).await?; rss_step.update(RssStep::CreateSledPlan); info!(self.log, "Creating new allocation plan"); @@ -1155,7 +1071,7 @@ impl ServiceInner { init_trust_quorum( &self.log, - trust_quorum.clone(), + ctx.trust_quorum_handle.clone(), tq_members.clone(), rack_id, ) @@ -1163,7 +1079,7 @@ impl ServiceInner { Some(InitialTrustQuorumConfig { members: tq_members.into_iter().collect(), - coordinator: trust_quorum.baseboard_id().clone(), + coordinator: ctx.trust_quorum_handle.baseboard_id().clone(), }) } else { None @@ -1187,7 +1103,7 @@ impl ServiceInner { }; info!(self.log, "Writing initial network configuration to bootstore"); rss_step.update(RssStep::InitialNetworkConfigUpdate); - bootstore + ctx.bootstore_node_handle .update_network_config( EarlyNetworkConfigEnvelope::from(&system_networking_config) .serialize_to_bootstore_with_generation( @@ -1248,7 +1164,7 @@ impl ServiceInner { "Writing final system networking configuration to bootstore", ); rss_step.update(RssStep::FinalNetworkConfigUpdate); - bootstore + ctx.bootstore_node_handle .update_network_config( EarlyNetworkConfigEnvelope::from(&system_networking_config) .serialize_to_bootstore_with_generation( @@ -1397,13 +1313,9 @@ impl ServiceInner { ) .await?; - // Finally, mark that we've completed executing the plans and handed off to nexus. - let mut ledger = Ledger::::new_with( - &self.log, - completed_marker_paths.clone(), - RssCompleteMarker::default(), - ); - ledger.commit().await?; + // Finally, mark that we've completed executing the plans and handed off + // to nexus. + ctx.write_rss_completed_ledger(&self.log).await?; Ok(()) } @@ -1445,13 +1357,8 @@ async fn init_trust_quorum( break; } - let mut still_waiting = String::new(); - for member in members.difference(&status.acked_prepares) { - still_waiting.push_str(&member.to_string()); - still_waiting.push(','); - } - let _ = still_waiting.strip_suffix(","); - + let still_waiting = + itertools::join(members.difference(&status.acked_prepares), ","); info!( log, "RSS: Trust quorum coordinator waiting for PrepareAcks"; diff --git a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs index d0d85bee1bf..7b342d94227 100644 --- a/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs +++ b/sled-agent/src/bootstrap/http_entrypoints_lockstep.rs @@ -18,16 +18,21 @@ use bootstrap_agent_lockstep_api::BootstrapAgentLockstepApi; use bootstrap_agent_lockstep_api::bootstrap_agent_lockstep_api_mod; use bootstrap_agent_lockstep_types::BaseboardIds; use bootstrap_agent_lockstep_types::BootstrapIpOfBaseboardId; +use bootstrap_agent_lockstep_types::MultirackJoinRequest; use bootstrap_agent_lockstep_types::RackInitializeRequest; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig; use bootstrap_agent_lockstep_types::ReplicatedNetworkConfigContents; +use dropshot::ClientErrorStatusCode; use dropshot::{ ApiDescription, HttpError, HttpResponseOk, RequestContext, TypedBody, }; +use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; +use sled_agent_bootstrap_common::RssContext; use sled_agent_config_reconciler::InternalDisksReceiver; use sled_agent_measurements::MeasurementsHandle; +use sled_agent_multirack_join::MultirackJoinServiceState; use sled_agent_rack_setup::RackInitializeRequestParams; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; @@ -42,26 +47,40 @@ pub(crate) struct BootstrapServerContext { pub(crate) internal_disks_rx: InternalDisksReceiver, pub(crate) bootstore_node_handle: bootstore::NodeHandle, pub(crate) rss_access: RssAccess, - pub(crate) sprockets: SprocketsConfig, + pub(crate) sprockets_config: SprocketsConfig, pub(crate) trust_quorum_handle: trust_quorum::NodeTaskHandle, pub(crate) measurements: Arc, } +impl From<&BootstrapServerContext> for RssContext { + fn from(value: &BootstrapServerContext) -> Self { + RssContext { + base_log: value.base_log.clone(), + global_zone_bootstrap_ip: value.global_zone_bootstrap_ip, + internal_disks_rx: value.internal_disks_rx.clone(), + bootstore_node_handle: value.bootstore_node_handle.clone(), + sprockets_config: value.sprockets_config.clone(), + trust_quorum_handle: value.trust_quorum_handle.clone(), + measurements: value.measurements.clone(), + } + } +} + impl BootstrapServerContext { + /// This is mutually exclusive with `start_multirack_join`. pub(super) fn start_rack_initialize( &self, request: RackInitializeRequestParams, ) -> Result { - self.rss_access.start_initializing( - &self.base_log, - self.sprockets.clone(), - self.global_zone_bootstrap_ip, - &self.internal_disks_rx, - self.measurements.clone(), - &self.bootstore_node_handle, - &self.trust_quorum_handle, - request, - ) + self.rss_access.start_initializing(self.into(), request) + } + + /// This is mutually exclusive with `start_rack_initialize`. + pub(super) fn start_multirack_join( + &self, + request: MultirackJoinRequest, + ) -> Result { + self.rss_access.start_multirack_join(self.into(), request) } } @@ -146,4 +165,33 @@ impl BootstrapAgentLockstepApi for BootstrapAgentLockstepImpl { Ok(HttpResponseOk(BaseboardIds { data })) } + + async fn multirack_join( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + let request = body.into_inner(); + let id = ctx + .start_multirack_join(request) + .map_err(|err| HttpError::for_bad_request(None, err.to_string()))?; + Ok(HttpResponseOk(id)) + } + + async fn multirack_join_state( + rqctx: RequestContext, + ) -> Result, HttpError> { + let ctx = rqctx.context(); + let state = + ctx.rss_access.get_multirack_join_state().map_err(|_| { + HttpError::for_client_error( + Some("Conflict".to_string()), + ClientErrorStatusCode::CONFLICT, + "Cannot run multirack join: RSS has been run on this rack" + .to_string(), + ) + })?; + + Ok(HttpResponseOk(state)) + } } diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index bd11b06719e..708666cff94 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -4,21 +4,21 @@ //! Internal API for rack-level bootstrap agent operations. -use crate::bootstrap::rss_handle::RssHandle; -use bootstore::schemes::v0 as bootstore; +use crate::bootstrap::rss_handle::run_rss; +use bootstrap_agent_lockstep_types::MultirackJoinRequest; use bootstrap_agent_lockstep_types::RackOperationStatus; use bootstrap_agent_lockstep_types::RssStep; use dropshot::HttpError; +use omicron_uuid_kinds::MultirackJoinUuid; use omicron_uuid_kinds::RackInitUuid; -use sled_agent_config_reconciler::InternalDisksReceiver; -use sled_agent_measurements::MeasurementsHandle; +use sled_agent_bootstrap_common::RssContext; +use sled_agent_multirack_join::MultirackJoinServiceError; +use sled_agent_multirack_join::MultirackJoinServiceHandle; +use sled_agent_multirack_join::MultirackJoinServiceState; use sled_agent_rack_setup::RackInitializeRequestParams; use sled_agent_rack_setup::SetupServiceError; -use slog::Logger; use slog_error_chain::InlineErrorChain; -use sprockets_tls::keys::SprocketsConfig; use std::mem; -use std::net::Ipv6Addr; use std::sync::Arc; use std::sync::Mutex; use tokio::sync::oneshot; @@ -35,6 +35,14 @@ pub enum RssAccessError { InitializationPanicked, #[error("RSS is already initialized")] AlreadyInitialized, + #[error("Multirack join in progress")] + MultirackJoinInProgress, + #[error("Multirack join failed: {message}")] + MultirackJoinFailed { message: String }, + #[error("MultirackJoin panicked")] + MultirackJoinPanicked, + #[error("Already part of a multirack cluster")] + MultirackJoinCompleted, } impl RssAccessError { @@ -48,6 +56,12 @@ impl RssAccessError { } RssAccessError::InitializationPanicked => "InitializationPanicked", RssAccessError::AlreadyInitialized => "AlreadyInitialized", + RssAccessError::MultirackJoinInProgress => { + "MultirackJoinInProgress" + } + RssAccessError::MultirackJoinFailed { .. } => "MultirackJoinFailed", + RssAccessError::MultirackJoinPanicked => "MultirackJoinPanicked", + RssAccessError::MultirackJoinCompleted => "MultirackJoinCompleted", } } } @@ -57,13 +71,13 @@ impl From for HttpError { // All variants of RssAccessError report states that conflict with the // requested rack operation in some fashion: // - // * `StillInitializing`/`StillResetting` mean that an operation is in - // progress. - // * `AlreadyInitialized`/`AlreadyReset` mean that we're already at the - // requested end state. (Note that we don't try and be idempotent here - // -- to do so we'd have to ensure the actual RSS config is the same, - // which is tricky. Instead, clients should look at the error code to - // determine what to do.) + // * `StillInitializing`/`MultirackJoinInProgress` mean that an + // operation is in progress. + // * `AlreadyInitialized`/`MultirackJoinCompleted` mean that we're + // already at the requested end state. (Note that we don't try and + // be idempotent here -- to do so we'd have to ensure the actual RSS + // config is the same, which is tricky. Instead, clients should look + // at the error code to determine what to do.) // * The other states are terminal states that need operator intervention. // // We map all of these states to 409 Conflict errors, and (since this is @@ -76,6 +90,9 @@ impl From for HttpError { } } +/// A mechanism for accessing rack setup related functionality. We're +/// using `RSS` as a catch all here. This type allows access to both rack +/// initialization and mulitrack regional cluster join services. #[derive(Clone)] pub(crate) struct RssAccess { // Note: The `Mutex` here is a std mutex, not a tokio mutex, and thus not @@ -99,6 +116,7 @@ impl RssAccess { let mut status = self.status.lock().unwrap(); match &mut *status { + RssStatus::Uninitialized => RackOperationStatus::Uninitialized, RssStatus::Initializing { id, completion, step_rx } => { let id = *id; // This is our only chance to notice the initialization task has @@ -137,25 +155,56 @@ impl RssAccess { RssStatus::InitializationPanicked { id } => { RackOperationStatus::InitializationPanicked { id: *id } } - RssStatus::Uninitialized => RackOperationStatus::Uninitialized, + RssStatus::MultirackJoinInProgress { id, .. } => { + RackOperationStatus::MultirackJoinInProgress { id: *id } + } + RssStatus::MultirackJoinCompleted { id } => { + RackOperationStatus::MultirackJoinCompleted { id: *id } + } + RssStatus::MultirackJoinFailed { id, err } => { + RackOperationStatus::MultirackJoinFailed { + id: *id, + message: InlineErrorChain::new(err).to_string(), + } + } + RssStatus::MultirackJoinPanicked { id } => { + RackOperationStatus::MultirackJoinPanicked { id: *id } + } } } - #[allow(clippy::too_many_arguments)] pub(crate) fn start_initializing( &self, - parent_log: &Logger, - sprockets: SprocketsConfig, - global_zone_bootstrap_ip: Ipv6Addr, - internal_disks_rx: &InternalDisksReceiver, - measurements: Arc, - bootstore_node_handle: &bootstore::NodeHandle, - trust_quorum_handle: &trust_quorum::NodeTaskHandle, + ctx: RssContext, request: RackInitializeRequestParams, ) -> Result { let mut status = self.status.lock().unwrap(); match &*status { + RssStatus::Uninitialized => { + let (completion_tx, completion) = oneshot::channel(); + let id = RackInitUuid::new_v4(); + let (step_tx, step_rx) = watch::channel(RssStep::Requested); + *status = RssStatus::Initializing { id, completion, step_rx }; + mem::drop(status); + let status = Arc::clone(&self.status); + tokio::spawn(async move { + let result = run_rss(ctx, request, step_tx).await; + let new_status = match result { + Ok(()) => RssStatus::Initialized { id: Some(id) }, + Err(err) => RssStatus::InitializationFailed { id, err }, + }; + + // Order here is critical: store the new status in the + // shared mutex _before_ signaling on the channel that + // initialization has completed; otherwise, callers waiting + // on the channel could see an incomplete status. + *status.lock().unwrap() = new_status; + _ = completion_tx.send(()); + }); + Ok(id) + } + RssStatus::Initializing { .. } => { Err(RssAccessError::StillInitializing) } @@ -171,48 +220,142 @@ impl RssAccess { Err(RssAccessError::InitializationPanicked) } + RssStatus::MultirackJoinInProgress { .. } => { + Err(RssAccessError::MultirackJoinInProgress) + } + RssStatus::MultirackJoinCompleted { .. } => { + Err(RssAccessError::MultirackJoinCompleted) + } + RssStatus::MultirackJoinFailed { err, .. } => { + Err(RssAccessError::MultirackJoinFailed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::MultirackJoinPanicked { .. } => { + Err(RssAccessError::MultirackJoinPanicked) + } + } + } + + pub(crate) fn start_multirack_join( + &self, + ctx: RssContext, + request: MultirackJoinRequest, + ) -> Result { + let mut status = self.status.lock().unwrap(); + + match &mut *status { RssStatus::Uninitialized => { - let (completion_tx, completion) = oneshot::channel(); - let id = RackInitUuid::new_v4(); - let (step_tx, step_rx) = watch::channel(RssStep::Requested); - *status = RssStatus::Initializing { id, completion, step_rx }; + let id = MultirackJoinUuid::new_v4(); + let MultirackJoinServiceHandle { + join_handle, + input_tx, + output_rx, + } = MultirackJoinServiceHandle::spawn(ctx, request); + *status = RssStatus::MultirackJoinInProgress { + id, + input_tx, + output_rx, + }; mem::drop(status); - let parent_log = parent_log.clone(); - let internal_disks_rx = internal_disks_rx.clone(); - let bootstore_node_handle = bootstore_node_handle.clone(); let status = Arc::clone(&self.status); - let trust_quorum_handle = trust_quorum_handle.clone(); + + // Spawn a task that waits for the multirack join service task to exit tokio::spawn(async move { - let result = rack_initialize( - &parent_log, - sprockets, - global_zone_bootstrap_ip, - internal_disks_rx, - measurements, - bootstore_node_handle, - trust_quorum_handle, - request, - step_tx, - ) - .await; - let new_status = match result { - Ok(()) => RssStatus::Initialized { id: Some(id) }, - Err(err) => RssStatus::InitializationFailed { id, err }, + let new_status = match join_handle.await { + Ok(Ok(())) => { + RssStatus::MultirackJoinCompleted { id: Some(id) } + } + Ok(Err(err)) => { + RssStatus::MultirackJoinFailed { id, err } + } + Err(_) => RssStatus::MultirackJoinPanicked { id }, }; - - // Order here is critical: store the new status in the - // shared mutex _before_ signaling on the channel that - // initialization has completed; otherwise, callers waiting - // on the channel could see an incomplete status. *status.lock().unwrap() = new_status; - _ = completion_tx.send(()); }); + Ok(id) } + + RssStatus::Initializing { .. } => { + Err(RssAccessError::StillInitializing) + } + RssStatus::Initialized { .. } => { + Err(RssAccessError::AlreadyInitialized) + } + RssStatus::InitializationFailed { err, .. } => { + Err(RssAccessError::InitializationFailed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::InitializationPanicked { .. } => { + Err(RssAccessError::InitializationPanicked) + } + + RssStatus::MultirackJoinInProgress { id, input_tx, .. } => { + // We allow updating the input so that we don't always require a + // clean slate like in RSS. + input_tx.send_if_modified(|saved| { + if &request != saved { + *saved = request; + true + } else { + false + } + }); + Ok(*id) + } + RssStatus::MultirackJoinCompleted { .. } => { + Err(RssAccessError::MultirackJoinCompleted) + } + RssStatus::MultirackJoinFailed { err, .. } => { + Err(RssAccessError::MultirackJoinFailed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::MultirackJoinPanicked { .. } => { + Err(RssAccessError::MultirackJoinPanicked) + } + } + } + + pub(crate) fn get_multirack_join_state( + &self, + ) -> Result { + let mut status = self.status.lock().unwrap(); + + match &mut *status { + RssStatus::Uninitialized => { + Ok(MultirackJoinServiceState::Uninitialized) + } + + RssStatus::Initializing { .. } => Err(RssConflictError), + RssStatus::Initialized { .. } => Err(RssConflictError), + RssStatus::InitializationFailed { .. } => Err(RssConflictError), + RssStatus::InitializationPanicked { .. } => Err(RssConflictError), + + RssStatus::MultirackJoinInProgress { output_rx, .. } => { + let state = output_rx.borrow().clone(); + Ok(state) + } + RssStatus::MultirackJoinCompleted { .. } => { + Ok(MultirackJoinServiceState::Completed) + } + RssStatus::MultirackJoinFailed { err, .. } => { + Ok(MultirackJoinServiceState::Failed { + message: InlineErrorChain::new(err).to_string(), + }) + } + RssStatus::MultirackJoinPanicked { .. } => { + Ok(MultirackJoinServiceState::TaskPanicked) + } } } } +// RSS was run, and not the multirack join service +pub struct RssConflictError; + enum RssStatus { // Our two main primary states. Uninitialized, @@ -241,32 +384,29 @@ enum RssStatus { InitializationPanicked { id: RackInitUuid, }, -} -#[allow(clippy::too_many_arguments)] -async fn rack_initialize( - parent_log: &Logger, - sprockets: SprocketsConfig, - global_zone_bootstrap_ip: Ipv6Addr, - internal_disks_rx: InternalDisksReceiver, - measurements: Arc, - bootstore_node_handle: bootstore::NodeHandle, - trust_quorum_handle: trust_quorum::NodeTaskHandle, - request: RackInitializeRequestParams, - step_tx: watch::Sender, -) -> Result<(), SetupServiceError> { - RssHandle::run_rss( - parent_log, - sprockets, - request, - global_zone_bootstrap_ip, - internal_disks_rx, - measurements, - bootstore_node_handle, - trust_quorum_handle, - step_tx, - ) - .await + // Tranistory states (which we may be in for a long time, even on human time + // scales, but should eventually leave). + MultirackJoinInProgress { + id: MultirackJoinUuid, + input_tx: watch::Sender, + output_rx: watch::Receiver, + }, + + MultirackJoinCompleted { + // We can either be joined on startup (in which case `id` is None) or + // because join has completed (in which case `id` is Some). + id: Option, + }, + + // Terminal failure states; these require support intervention. + MultirackJoinFailed { + id: MultirackJoinUuid, + err: MultirackJoinServiceError, + }, + MultirackJoinPanicked { + id: MultirackJoinUuid, + }, } #[cfg(test)] diff --git a/sled-agent/src/bootstrap/rss_handle.rs b/sled-agent/src/bootstrap/rss_handle.rs index e4ddc7b72cf..dd7492eb370 100644 --- a/sled-agent/src/bootstrap/rss_handle.rs +++ b/sled-agent/src/bootstrap/rss_handle.rs @@ -6,14 +6,13 @@ use super::sprockets_client::SprocketsClient; use super::sprockets_client::SprocketsClientError; -use bootstore::schemes::v0 as bootstore; use bootstrap_agent_lockstep_types::RssStep; use futures::StreamExt; use futures::stream::FuturesUnordered; use omicron_common::backoff::BackoffError; use omicron_common::backoff::retry_notify; use omicron_common::backoff::retry_policy_local; -use sled_agent_config_reconciler::InternalDisksReceiver; +use sled_agent_bootstrap_common::RssContext; use sled_agent_measurements::MeasurementsHandle; use sled_agent_rack_setup::LocalBootstrapAgent; use sled_agent_rack_setup::RackInitializeRequestParams; @@ -22,60 +21,25 @@ use sled_agent_rack_setup::SetupServiceError; use sled_agent_types::sled::StartSledAgentRequest; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; -use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::sync::Arc; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; -use tokio::task::JoinHandle; -pub(super) struct RssHandle { - _rss: RackSetupService, - task: JoinHandle<()>, -} - -impl Drop for RssHandle { - fn drop(&mut self) { - // NOTE: Ideally, with async drop, we'd await completion of the our task - // handler. - // - // Without that option, we instead opt to simply cancel the task to - // ensure it does not remain alive beyond the handle itself. - self.task.abort(); - } -} - -impl RssHandle { - /// Executes the rack setup service until it has completed - #[allow(clippy::too_many_arguments)] - pub(super) async fn run_rss( - log: &Logger, - sprockets: SprocketsConfig, - config: RackInitializeRequestParams, - our_bootstrap_address: Ipv6Addr, - internal_disks_rx: InternalDisksReceiver, - measurements: Arc, - bootstore: bootstore::NodeHandle, - trust_quorum: trust_quorum::NodeTaskHandle, - step_tx: watch::Sender, - ) -> Result<(), SetupServiceError> { - let (tx, rx) = rss_channel(sprockets, measurements.clone()); - - let rss = RackSetupService::new( - log.new(o!("component" => "RSS")), - config, - internal_disks_rx, - tx, - our_bootstrap_address, - bootstore, - trust_quorum, - step_tx, - ); - let log = log.new(o!("component" => "BootstrapAgentRssHandler")); - rx.await_local_rss_request(&log).await; - rss.join().await - } +/// Executes the rack setup service until it has completed +pub(super) async fn run_rss( + ctx: RssContext, + config: RackInitializeRequestParams, + step_tx: watch::Sender, +) -> Result<(), SetupServiceError> { + let (tx, rx) = + rss_channel(ctx.sprockets_config.clone(), ctx.measurements.clone()); + + let log = ctx.base_log.new(o!("component" => "BootstrapAgentRssHandler")); + let rss = RackSetupService::new(ctx, config, tx, step_tx); + rx.await_local_rss_request(&log).await; + rss.join().await } // Send a message to start a sled agent via bootstrap agent client diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 266d93b47da..a7891fb0527 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -209,7 +209,7 @@ impl Server { internal_disks_rx, bootstore_node_handle: long_running_task_handles.bootstore.clone(), rss_access, - sprockets: config.sprockets.clone(), + sprockets_config: config.sprockets.clone(), trust_quorum_handle: long_running_task_handles.trust_quorum.clone(), measurements: long_running_task_handles.measurements.clone(), }; diff --git a/trust-quorum/types/versions/src/initial/status.rs b/trust-quorum/types/versions/src/initial/status.rs index 8fa7f425188..ce5ffd94955 100644 --- a/trust-quorum/types/versions/src/initial/status.rs +++ b/trust-quorum/types/versions/src/initial/status.rs @@ -23,7 +23,7 @@ pub enum CommitStatus { } /// Status of the node coordinating the reconfiguration or LRTQ upgrade. -#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] pub struct CoordinatorStatus { pub config: Configuration, pub acked_prepares: BTreeSet, diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index f0368dbd472..ebad8af5c4e 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -70,6 +70,7 @@ impl_typed_uuid_kinds! { LoopbackAddress = {}, MulticastGroup = {}, Measurement = {}, + MultirackJoin = {}, Mupdate = {}, MupdateOverride = {}, // `OmicronSledConfig`s do not themselves contain IDs, but we generate IDs diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 1e5a6237304..91b8bbcd75e 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -313,6 +313,49 @@ fn draw_rack_status_details_popup( style::plain_text(), )])); } + Ok(RackOperationStatus::MultirackJoinInProgress { id }) => { + body.lines.push(Line::from(vec![ + status, + Span::styled("Multirack Join In Progress", style::plain_text()), + ])); + body.lines.push(Line::from(vec![Span::styled( + format!("Current operation ID: {}", id), + style::plain_text(), + )])); + } + Ok(RackOperationStatus::MultirackJoinCompleted { id }) => { + body.lines.push(Line::from(vec![ + status, + Span::styled("Multirack Join Completed", style::plain_text()), + ])); + if let Some(id) = id { + body.lines.push(Line::from(vec![Span::styled( + format!("Last multirack join operation ID: {}", id), + style::plain_text(), + )])); + } + } + Ok(RackOperationStatus::MultirackJoinFailed { id, message }) => { + body.lines.push(Line::from(vec![ + status, + Span::styled("Multirack Join Failed", style::plain_text()), + ])); + body.lines.push(Line::from(vec![Span::styled( + format!("Last multirack join operation ID: {}", id), + style::plain_text(), + )])); + push_text_lines(message, prefix, &mut body.lines); + } + Ok(RackOperationStatus::MultirackJoinPanicked { id }) => { + body.lines.push(Line::from(vec![ + status, + Span::styled("Multirack Join Panicked", style::plain_text()), + ])); + body.lines.push(Line::from(vec![Span::styled( + format!("Last multirack join operation ID: {}", id), + style::plain_text(), + )])); + } Err(message) => { body.lines.push(Line::from(vec![ status, @@ -494,6 +537,16 @@ fn rss_config_text<'a>( RackOperationStatus::InitializationFailed { .. } | RackOperationStatus::InitializationPanicked { .. }, ) => Span::styled("Initialization Failed", bad_style), + Ok(RackOperationStatus::MultirackJoinInProgress { .. }) => { + Span::styled("Multirack Join In Progress", warn_style) + } + Ok(RackOperationStatus::MultirackJoinCompleted { .. }) => { + Span::styled("Multirack Join Completed", ok_style) + } + Ok( + RackOperationStatus::MultirackJoinFailed { .. } + | RackOperationStatus::MultirackJoinPanicked { .. }, + ) => Span::styled("Multirack Join Failed", bad_style), Err(_) => Span::styled("Unknown", bad_style), }; diff --git a/wicketd-commission-types/versions/src/initial/rack_setup.rs b/wicketd-commission-types/versions/src/initial/rack_setup.rs index ce7dc8b8f36..f0efc9c446a 100644 --- a/wicketd-commission-types/versions/src/initial/rack_setup.rs +++ b/wicketd-commission-types/versions/src/initial/rack_setup.rs @@ -734,8 +734,8 @@ impl RackOperationKind { /// The well-known kind for a rack initialization operation. pub const INITIALIZE: Self = Self(Cow::Borrowed("initialize")); - /// The well-known kind for a rack reset operation. - pub const RESET: Self = Self(Cow::Borrowed("reset")); + /// The well-known kind for a multirack join operation + pub const MULTIRACK_JOIN: Self = Self(Cow::Borrowed("multirack-join")); /// Returns the operation kind as a string. pub fn as_str(&self) -> &str { diff --git a/wicketd/src/commission/conversions.rs b/wicketd/src/commission/conversions.rs index 1c6569b9a0b..182f01cdff8 100644 --- a/wicketd/src/commission/conversions.rs +++ b/wicketd/src/commission/conversions.rs @@ -747,6 +747,45 @@ pub(crate) fn rack_setup_status_to_ct( rack_state: RackState::Initialized, operation: None, }, + B::MultirackJoinInProgress { id } => RackSetupStatus { + rack_state: RackState::Other, + operation: Some(RackOperation { + kind: RackOperationKind::MULTIRACK_JOIN, + id: id.into_untyped_uuid(), + state: RackOperationState::InProgress { current_step: None }, + }), + }, + B::MultirackJoinCompleted { id: Some(id) } => RackSetupStatus { + rack_state: RackState::Initialized, + operation: Some(RackOperation { + kind: RackOperationKind::MULTIRACK_JOIN, + id: id.into_untyped_uuid(), + state: RackOperationState::Completed, + }), + }, + B::MultirackJoinCompleted { id: None } => RackSetupStatus { + rack_state: RackState::Initialized, + operation: None, + }, + B::MultirackJoinFailed { id, message } => RackSetupStatus { + rack_state: RackState::Other, + operation: Some(RackOperation { + kind: RackOperationKind::MULTIRACK_JOIN, + id: id.into_untyped_uuid(), + state: RackOperationState::Failed { + message, + failed_step: None, + }, + }), + }, + B::MultirackJoinPanicked { id } => RackSetupStatus { + rack_state: RackState::Other, + operation: Some(RackOperation { + kind: RackOperationKind::MULTIRACK_JOIN, + id: id.into_untyped_uuid(), + state: RackOperationState::Panicked, + }), + }, } }