BMC: Added CLI commands runner and Serial Console access. (#419)

* BMC: Added CLI commands runner and Serial Console access.

BMC's CLI commands can be run using the bmc.RunCLICommand method,
which connects to the CLI via SSH. The method blocks until the command
ends or a timeout happens, whatever comes first. The command's output is
copied into the stdout & stderr string vars. The underlying SSH session
is created and closed internally after the command finishes.

The new method OpenSerialConsole returns a piped reader and writer
interfaces that can be used to interactively receive/send commands/text
to/from the serial console. Since the serial console is tunneled in a
SSH session, the CloseSerialConsole method must be called to release
that session to prevent leaks that would prevent CLI commands to fail
due to maximum concurrent SSH sessions reached in the BMC.

Also, added SSH port as parameter for BMC's New() method

* Addressed comments from Nikita.

* Added host on glog traces and errors.

* Updated UTs to the new error messages.

Author: greyerof

README.md

@@ -96,6 +96,59 @@ exit status 1
 ```
 Please refer to the [secret pkg](./pkg/secret/secret.go)'s use of the validate method for more information.
 
+### BMC Package
+The BMC package can be used to access the BMC's Redfish API, run BMC's CLI commands or getting the systems' serial console. Credentials for both Redfish and SSH user, the SSH port and timeouts need to be passed as parameters of the New() method. E.g.
+
+```
+redfishUser := bmc.User{Name: "redfishuser1", Password: "redfishpass1"}
+sshUser := bmc.User{Name: "sshuser1", Password: "sshpass1"}
+timeOuts := bmc.TimeOuts{Redfish: 10*time.Second, SSH: 10*time.Second}
+
+bmc, err := bmc.New("1.2.3.4", redfishUser, sshUser, 22, timeOuts)
+```
+
+You can check an example program for the BMC package [here](usage/bmc/bmc.go).
+
+#### BMC's Redfish API
+The access to BMC's Redfish API is done by methods that encapsulate the underlaying HTTP calls made by the external gofish library. The redfish system index is defaulted to 0, but it can be changed with `SetSystemIndex()`:
+```
+const systemIndex = 3
+err = bmc.SetSystemIndex(systemIndex)
+if err != nil {
+    ...
+}
+
+manufacturer, err := bmc.SystemManufacturer()
+if err != nil {
+    ...
+}
+
+fmt.Printf("System %d's manufacturer: %v", systemIndex, manufacturer)
+
+```
+
+#### BMC's CLI
+The method `RunCLICommand` has been implemented to run CLI commands.
+```
+func (bmc *BMC) RunCLICommand(cmd string, combineOutput bool, timeout time.Duration) stdout string, stderr string, err error)
+```
+This method is not interactive: it blocks the caller until the command ends, copying its output into stdout and stderr strings.
+
+#### Serial Console
+The method `OpenSerialConsole` can be used to get the systems's serial console, which is tunneled in the an underlaying SSH session.
+```
+func (bmc *BMC) OpenSerialConsole(openConsoleCliCmd string) (io.Reader, io.WriteCloser, error)
+```
+The user gets a (piped) reader and writer interfaces in order to read the output or write custom input (like CLI commands) in a interactive fashion.
+A use case for this is a test case that needs to wait for some pattern to appear in the system's serial console after rebooting the system.
+
+The `openConsoleCliCmd` is the command that will be sent to the BMC's (SSH'd) CLI to open the serial console. In case the user doesn't know the command,
+it can be left empty. In that case, there's a best effort mechanism that will try to guess the CLI command based on the system's manufacturer, which will
+be internally retrieved using the Redfish API.
+
+It's important to close the serial console using the method `bmc.CloseSerialConsole()`, which closes the underlying SSH session. Otherwise, BMC's can reach
+the maximum number of concurrent SSH sessions making other (SSH'd CLI) commands to fail. See an example program [here](usage/bmc/bmc.go).
+
 # eco-goinfra - How to contribute
 
 The project uses a development method - forking workflow

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/stmcginnis/gofish v0.15.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vmware-tanzu/velero v1.12.1
+	golang.org/x/crypto v0.23.0
 	open-cluster-management.io/api v0.12.0
 )
 
@@ -192,7 +193,6 @@ require (
 	go.mongodb.org/mongo-driver v1.11.1 // indirect
 	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go4.org v0.0.0-20200104003542-c7e774b10ea0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
 	golang.org/x/oauth2 v0.15.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect

pkg/bmc/bmc.go

@@ -1,18 +1,24 @@
 package bmc
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/golang/glog"
 	"github.com/stmcginnis/gofish"
 	"github.com/stmcginnis/gofish/redfish"
+	"golang.org/x/crypto/ssh"
 )
 
 const (
 	defaultTimeOut = 5 * time.Second
+
+	manufacturerDell = "Dell Inc."
+	manufacturerHPE  = "HPE"
 )
 
 var (
@@ -21,6 +27,12 @@ var (
 		Redfish: defaultTimeOut,
 		SSH:     defaultTimeOut,
 	}
+
+	// CLI command to get the serial console (virtual serial port).
+	cliCmdSerialConsole = map[string]string{
+		manufacturerHPE:  "VSP",
+		manufacturerDell: "console com2",
+	}
 )
 
 // User holds the Name and Password for a user (ssh/redfish).
@@ -44,14 +56,17 @@ type BMC struct {
 	host        string
 	redfishUser User
 	sshUser     User
+	sshPort     uint16
 	systemIndex int
 
 	timeOuts TimeOuts
+
+	sshSessionForSerialConsole *ssh.Session
 }
 
 // New returns a new BMC struct. The default system index to be used in redfish requests is 0.
 // Use SetSystemIndex to modify it.
-func New(host string, redfishUser, sshUser User, timeOuts TimeOuts) (*BMC, error) {
+func New(host string, redfishUser, sshUser User, sshPort uint16, timeOuts TimeOuts) (*BMC, error) {
 	glog.V(100).Infof("Initializing new BMC structure with the following params: %s, %v, %v, %v (system index = 0)",
 		host, redfishUser, sshUser, timeOuts)
 
@@ -72,6 +87,10 @@ func New(host string, redfishUser, sshUser User, timeOuts TimeOuts) (*BMC, error
 		errMsgs = append(errMsgs, "ssh user's name is empty")
 	}
 
+	if sshPort == 0 {
+		errMsgs = append(errMsgs, "ssh port is zero")
+	}
+
 	if sshUser.Password == "" {
 		errMsgs = append(errMsgs, "ssh user's password is empty")
 	}
@@ -105,6 +124,7 @@ func New(host string, redfishUser, sshUser User, timeOuts TimeOuts) (*BMC, error
 		host:        host,
 		redfishUser: redfishUser,
 		sshUser:     sshUser,
+		sshPort:     sshPort,
 		timeOuts:    timeOuts,
 		systemIndex: 0,
 	}, nil
@@ -347,3 +367,202 @@ func redfishGetSystemSecureBoot(redfishClient *gofish.APIClient, systemIndex int
 
 	return sboot, nil
 }
+
+// CreateCLISSHSession creates a ssh Session to the host.
+func (bmc *BMC) CreateCLISSHSession() (*ssh.Session, error) {
+	glog.V(100).Infof("Creating SSH session to run commands in the BMC's CLI.")
+
+	config := &ssh.ClientConfig{
+		User: bmc.sshUser.Name,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(bmc.sshUser.Password),
+			ssh.KeyboardInteractive(func(user, instruction string, questions []string,
+				echos []bool) (answers []string, err error) {
+				answers = make([]string, len(questions))
+				// The second parameter is unused
+				for n := range questions {
+					answers[n] = bmc.sshUser.Password
+				}
+
+				return answers, nil
+			}),
+		},
+		Timeout:         bmc.timeOuts.SSH,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+
+	// Establish SSH connection
+	client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", bmc.host, bmc.sshPort), config)
+	if err != nil {
+		glog.V(100).Infof("Failed to connect to BMC's SSH server: %v", err)
+
+		return nil, fmt.Errorf("failed to connect to BMC's SSH server: %w", err)
+	}
+
+	// Create a session
+	session, err := client.NewSession()
+	if err != nil {
+		glog.V(100).Infof("Failed to create a new SSH session: %v", err)
+
+		return nil, fmt.Errorf("failed to create a new ssh session: %w", err)
+	}
+
+	return session, nil
+}
+
+// RunCLICommand runs a CLI command in the BMC's console. This method will block until the command
+// has finished, and its output is copied to stdout and/or stderr if applicable. If combineOutput is true,
+// stderr content is merged in stdout. The timeout param is used to avoid the caller to be stuck forever
+// in case something goes wrong or the command is stuck.
+func (bmc *BMC) RunCLICommand(cmd string, combineOutput bool, timeout time.Duration) (
+	stdout string, stderr string, err error) {
+	glog.V(100).Infof("Running CLI command in BMC's CLI: %s", cmd)
+
+	sshSession, err := bmc.CreateCLISSHSession()
+	if err != nil {
+		glog.V(100).Infof("Failed to connect to CLI: %v", err)
+
+		return "", "", fmt.Errorf("failed to connect to CLI: %w", err)
+	}
+
+	defer sshSession.Close()
+
+	var stdoutBuffer, stderrBuffer bytes.Buffer
+	if !combineOutput {
+		sshSession.Stdout = &stdoutBuffer
+		sshSession.Stderr = &stderrBuffer
+	}
+
+	var combinedOutput []byte
+
+	errCh := make(chan error)
+	go func() {
+		var err error
+		if combineOutput {
+			combinedOutput, err = sshSession.CombinedOutput(cmd)
+		} else {
+			err = sshSession.Run(cmd)
+		}
+		errCh <- err
+	}()
+
+	timeoutCh := time.After(timeout)
+
+	select {
+	case <-timeoutCh:
+		glog.V(100).Info("CLI command timeout")
+
+		return stdoutBuffer.String(), stderrBuffer.String(), fmt.Errorf("timeout running command")
+	case err := <-errCh:
+		glog.V(100).Info("Command run error: %v", err)
+
+		if err != nil {
+			return stdoutBuffer.String(), stderrBuffer.String(), fmt.Errorf("command run error: %w", err)
+		}
+	}
+
+	if combineOutput {
+		return string(combinedOutput), "", nil
+	}
+
+	return stdoutBuffer.String(), stderrBuffer.String(), nil
+}
+
+// OpenSerialConsole opens the serial console port. The console is tunneled in an underlying (CLI)
+// ssh session that is opened in the BMC's ssh server. If openConsoleCliCmd is
+// provided, it will be sent to the BMC's cli. Otherwise, a best effort will
+// be made to run the appropriate cli command based on the system manufacturer.
+func (bmc *BMC) OpenSerialConsole(openConsoleCliCmd string) (io.Reader, io.WriteCloser, error) {
+	glog.V(100).Infof("Opening serial console on %v.", bmc.host)
+
+	if bmc.sshSessionForSerialConsole != nil {
+		glog.V(100).Infof("There is already a serial console opened for %v's BMC. Use OpenSerialConsole() first.",
+			bmc.host)
+
+		return nil, nil, fmt.Errorf("there is already a serial console opened for %v's BMC", bmc.host)
+	}
+
+	cliCmd := openConsoleCliCmd
+	if cliCmd == "" {
+		// no cli command to get console port was provided, try to guess based on
+		// manufacturer.
+		manufacturer, err := bmc.SystemManufacturer()
+		if err != nil {
+			glog.V(100).Infof("Failed to get redifsh system manufacturer for %v: %v", bmc.host, err)
+
+			return nil, nil, fmt.Errorf("failed to get redfish system manufacturer for %v: %w", bmc.host, err)
+		}
+
+		var found bool
+		if cliCmd, found = cliCmdSerialConsole[manufacturer]; !found {
+			glog.V(100).Infof("CLI command to get serial console not found for manufacturer for %v: %v",
+				bmc.host, manufacturer)
+
+			return nil, nil, fmt.Errorf("cli command to get serial console not found for manufacturer for %v: %v",
+				bmc.host, manufacturer)
+		}
+	}
+
+	sshSession, err := bmc.CreateCLISSHSession()
+	if err != nil {
+		glog.V(100).Infof("Failed to create underlying ssh session for %v: %v", bmc.host, err)
+
+		return nil, nil, fmt.Errorf("failed to create underlying ssh session for %v: %w", bmc.host, err)
+	}
+
+	// Pipes need to be retrieved before session.Start()
+	reader, err := sshSession.StdoutPipe()
+	if err != nil {
+		glog.V(100).Infof("Failed to get stdout pipe from %v's ssh session: %v", bmc.host, err)
+
+		_ = sshSession.Close()
+
+		return nil, nil, fmt.Errorf("failed to get stdout pipe from %v's ssh session: %w", bmc.host, err)
+	}
+
+	writer, err := sshSession.StdinPipe()
+	if err != nil {
+		glog.V(100).Infof("Failed to get stdin pipe from from %v's ssh session: %w", bmc.host, err)
+
+		_ = sshSession.Close()
+
+		return nil, nil, fmt.Errorf("failed to get stdin pipe from %v's ssh session: %w", bmc.host, err)
+	}
+
+	err = sshSession.Start(cliCmd)
+	if err != nil {
+		glog.V(100).Infof("Failed to start CLI command %q on %v: %v", cliCmd, bmc.host, err)
+
+		_ = sshSession.Close()
+
+		return nil, nil, fmt.Errorf("failed to start serial console with cli command %q on %v: %w", cliCmd, bmc.host, err)
+	}
+
+	go func() { _ = sshSession.Wait() }()
+
+	bmc.sshSessionForSerialConsole = sshSession
+
+	return reader, writer, nil
+}
+
+// CloseSerialConsole closes the serial console's underlying ssh session.
+func (bmc *BMC) CloseSerialConsole() error {
+	glog.V(100).Infof("Closing serial console for %v.", bmc.host)
+
+	if bmc.sshSessionForSerialConsole == nil {
+		glog.V(100).Infof("No underlying ssh session found for %v. Please use OpenSerialConsole() first.", bmc.host)
+
+		return fmt.Errorf("no underlying ssh session found for %v", bmc.host)
+	}
+
+	err := bmc.sshSessionForSerialConsole.Close()
+	if err != nil {
+		glog.V(100).Infof("Failed to close underlying ssh session for %v: %v", bmc.host, err)
+
+		return fmt.Errorf("failed to close underlying ssh session for %v: %w", bmc.host, err)
+	}
+
+	bmc.sshSessionForSerialConsole = nil
+
+	return nil
+}