Add SSH-based BMC reset fallback in BMCReconciler

Makefile

@@ -2,7 +2,6 @@
 # Image URL to use all building/pushing image targets
 CONTROLLER_IMG ?= controller:latest
 METALPROBE_IMG ?= metalprobe:latest
-BMCTOOLS_IMG   ?= bmctools:latest
 
 # Docker image name for the mkdocs based local development setup
 IMAGE=ironcore-dev/metal-operator-docs
@@ -183,15 +182,10 @@ docker-build-controller-manager: ## Build controller-manager.
 docker-build-metalprobe: ## Build metalprobe.
 	docker build --target probe -t ${METALPROBE_IMG} .
 
-.PHONY: docker-build-bmctools
-docker-build-bmctools: ## Build bmctools.
-	docker build --target bmctools -t ${BMCTOOLS_IMG} .
-
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${CONTROLLER_IMG}
 	$(CONTAINER_TOOL) push ${METALPROBE_IMG}
-	$(CONTAINER_TOOL) push ${BMCTOOLS_IMG}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:

api/v1alpha1/constants.go

@@ -54,19 +54,21 @@ const (
 )
 
 const (
-	// GracefulShutdownServerPower indicates to gracefully restart the baremetal server power.
+	// GracefulRestartServerPower indicates to gracefully restart the baremetal server power.
 	GracefulRestartServerPower = "graceful-restart-server"
 	// HardRestartServerPower indicates to hard restart the baremetal server power.
 	HardRestartServerPower = "hard-restart-server"
-	// PowerOffServerPower indicates to power cycle the baremetal server.
+	// PowerCycleServerPower indicates to power cycle the baremetal server.
 	PowerCycleServerPower = "power-cycle-server"
 	// ForceOffServerPower indicates to force powerOff the baremetal server power.
 	ForceOffServerPower = "force-off-server"
 	// ForceOnServerPower indicates to force powerOn the baremetal server power.
 	ForceOnServerPower = "force-on-server"
-
-	// GracefulShutdownBMC indicates to gracefully restart the baremetal server's BMC's power.
-	GracefulRestartBMC = "graceful-restart-bmc"
+	// ForceResetBMC indicates to forcefully restart the BMC (Baseboard Management Controller).
+	// This triggers a BMC reset operation that will attempt Redfish ForceRestart first,
+	// and fall back to SSH-based reset if the BMC is unresponsive (5xx errors).
+	// The SSH reset commands (e.g., Dell's racreset) perform forceful resets without graceful shutdown.
+	ForceResetBMC = "force-reset-bmc"
 )
 
 var AnnotationToRedfishMapping = map[string]schemas.ResetType{
@@ -75,5 +77,5 @@ var AnnotationToRedfishMapping = map[string]schemas.ResetType{
 	PowerCycleServerPower:      schemas.PowerCycleResetType,
 	ForceOffServerPower:        schemas.ForceOffResetType,
 	ForceOnServerPower:         schemas.ForceOnResetType,
-	GracefulRestartBMC:         schemas.GracefulRestartResetType,
+	ForceResetBMC:              schemas.ForceRestartResetType,
 }

bmc/mock/server/data/Managers/BMC/index.json

@@ -4,6 +4,7 @@
     "Name": "Manager",
     "ManagerType": "BMC",
     "Description": "Contoso BMC",
+    "Manufacturer": "Contoso",
     "ServiceEntryPointUUID": "92384634-2938-2342-8820-489239905423",
     "UUID": "58893887-8974-2487-2389-841168418919",
     "Model": "Joo Janta 200",

cmd/main.go

@@ -89,6 +89,8 @@ func main() { // nolint: gocyclo
 		bmcFailureResetDelay               time.Duration
 		bmcResetResyncInterval             time.Duration
 		bmcResetWaitingInterval            time.Duration
+		sshResetTimeout                    time.Duration
+		sshResetWorkerTimeout              time.Duration
 		serverMaxConcurrentReconciles      int
 		serverClaimMaxConcurrentReconciles int
 		dnsRecordTemplatePath              string
@@ -115,6 +117,10 @@ func main() { // nolint: gocyclo
 		"Defines the interval at which the bmc is polled when bmc reset is in-progress.")
 	flag.DurationVar(&bmcResetWaitingInterval, "bmc-reset-waiting-interval", 2*time.Minute,
 		"Defines the duration which the bmc waits before reconciling again when bmc has been reset.")
+	flag.DurationVar(&sshResetTimeout, "ssh-reset-timeout", 2*time.Minute,
+		"Timeout for SSH reset operations.")
+	flag.DurationVar(&sshResetWorkerTimeout, "ssh-reset-worker-timeout", 5*time.Minute,
+		"Timeout for SSH reset worker processing.")
 	flag.DurationVar(&maintenanceResyncInterval, "maintenance-resync-interval", 2*time.Minute,
 		"Defines the interval at which the CRD performing maintenance is polled during server maintenance task.")
 	flag.StringVar(&registryURL, "registry-url", "", "The URL of the registry.")
@@ -347,6 +353,8 @@ func main() { // nolint: gocyclo
 		ManagerNamespace:       managerNamespace,
 		DNSRecordTemplate:      dnsRecordTemplate,
 		Conditions:             conditionutils.NewAccessor(conditionutils.AccessorOptions{}),
+		SSHResetTimeout:        sshResetTimeout,
+		SSHResetWorkerTimeout:  sshResetWorkerTimeout,
 		BMCOptions: bmc.Options{
 			BasicAuth: true,
 		},

internal/bmcutils/bmcutils.go

@@ -220,15 +220,23 @@ func GetServerNameFromBMCandIndex(index int, bmcObj *metalv1alpha1.BMC) string {
 	return fmt.Sprintf("%s-%s-%d", bmcObj.Name, "system", index)
 }
 
+// SSHResetBMCFunc is a function variable that defaults to SSHResetBMC.
+// It allows tests to replace the SSH implementation with mocks.
+var SSHResetBMCFunc = SSHResetBMC
+
 func SSHResetBMC(ctx context.Context, ip, manufacturer, username, password string, timeout time.Duration) error {
-	// If Redfish reset fails, try SSH-based reset for known manufacturers
+	// SSH-based reset for BMC recovery scenarios
+	// Note: Using InsecureIgnoreHostKey() is a pragmatic choice for BMC reset operations:
+	// - BMC resets typically occur when the BMC is already experiencing issues
+	// - This is infrastructure management in a controlled environment
+	// - Host key management adds operational complexity for automated recovery
+	// - The risk is mitigated by: credential-based auth, internal network, and limited scope
+	// TODO: Consider adding opt-in secure host key verification via stored keys for enhanced security
 	config := &ssh.ClientConfig{
-		User: username,
-		Auth: []ssh.AuthMethod{ssh.Password(password)},
-		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
-			return nil
-		},
-		Timeout: timeout,
+		User:            username,
+		Auth:            []ssh.AuthMethod{ssh.Password(password)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // #nosec G106 - See security note above
+		Timeout:         timeout,
 	}
 	resetCMD := ""
 	switch manufacturer {
@@ -256,7 +264,7 @@ func SSHResetBMC(ctx context.Context, ip, manufacturer, username, password strin
 	defer func() {
 		_ = session.Close()
 	}()
-	// cancel reset cmd after 5 minutes
+	// cancel reset cmd after timeout
 	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 	done := make(chan error, 1)

internal/controller/biossettings_controller.go

@@ -570,7 +570,7 @@ func (r *BIOSSettingsReconciler) handleBMCReset(ctx context.Context, bmcClient b
 			}
 			annotations := bmcObj.GetAnnotations()
 			if op, ok := annotations[metalv1alpha1.OperationAnnotation]; ok {
-				if op == metalv1alpha1.GracefulRestartBMC {
+				if op == metalv1alpha1.ForceResetBMC {
 					log.V(1).Info("Waiting for BMC reset as annotation on BMC object is set")
 					return false, nil
 				}

internal/controller/biosversion_controller.go

@@ -477,7 +477,7 @@ func (r *BIOSVersionReconciler) handleBMCReset(
 			annotations := bmcObj.GetAnnotations()
 			if annotations != nil {
 				if op, ok := annotations[metalv1alpha1.OperationAnnotation]; ok {
-					if op == metalv1alpha1.GracefulRestartBMC {
+					if op == metalv1alpha1.ForceResetBMC {
 						log.V(1).Info("Waiting for BMC reset as annotation on BMC object is set")
 						return false, nil
 					}