slemicro manual upgrade and node replacement support (rancher#231)

* slemicro upgrades

* add wait for ssh ready

* fix lint

* review comments

* update func name

* move scp after reboot

* mkdir /opt/data for path provisioner fix

* k3s skip enable fix

* lint fix

* lint

* lint

* moving mkdir /opt/data to test

* fix lint

* fix lint

* fix reboot timings

* lint

* reduce wait time

* add pod describe and logs

* typo fix

* linter

* lint

* lint

* testing delay changes

* add vars to cluster

* adding namespace for pod

* change error msg

* filename fix

* try

* adding debug log lines

* check if increasing timeout helps

* adding create /opt/data for agent nodes

* revert time increase

* re adding extra time for timeout

* update master server

* added return

* try reboot in lps for slemicro

* revert reboot

* adding more logging

* fix logs and remove create /var/../storage dir

* fix lint

* review comment fixes

* review comment fixes

* review fix

* lint fix

* reordered the nodes a bit

* reverting resource name in cluster object

* fix review comments

* review fixes

* comment fixes

* split the slemicro into separate func

* lint fix

* lint fix

Author: aganesh-suse

pkg/aws/ec2.go

@@ -213,6 +213,28 @@ func (c Client) StartInstance(instanceID string) error {
 	return nil
 }
 
+func (c Client) RebootInstance(instanceID string) error {
+	if instanceID == "" {
+		return shared.ReturnLogError("calling RebootInstance with empty instance ID, must send a valid instance ID")
+	}
+
+	input := &ec2.RebootInstancesInput{
+		InstanceIds: []*string{aws.String(instanceID)},
+	}
+
+	_, err := c.ec2.RebootInstances(input)
+	if err != nil {
+		return shared.ReturnLogError("failed to reboot instance %s: %v", instanceID, err)
+	}
+
+	startErr := c.waitForInstanceRunning(instanceID)
+	if startErr != nil {
+		return shared.ReturnLogError("timed out on reboot instance %s: %v", instanceID, startErr)
+	}
+
+	return nil
+}
+
 func (c Client) ReleaseElasticIps(ipAddress string) error {
 	if ipAddress == "" {
 		return shared.ReturnLogError("calling ReleaseElasticIps with empty ip address, must send a valid ip address")

pkg/testcase/clusterrestore.go

@@ -111,7 +111,7 @@ func newInstance(awsClient *aws.Client) (newServerName, newExternalIP string) {
 func installProduct(cluster *shared.Cluster, newClusterIP, version string) {
 	setConfigFile(cluster, newClusterIP)
 
-	installCmd := shared.GetInstallCmd(cluster.Config.Product, version, "server")
+	installCmd := shared.GetInstallCmd(cluster, version, "server")
 	if cluster.Config.Product == "k3s" {
 		skipInstall := fmt.Sprintf(" INSTALL_%s_SKIP_ENABLE=true ", strings.ToUpper(cluster.Config.Product))
 		installCmd = strings.Replace(installCmd, "sh", skipInstall+" "+"  sh", 1)

pkg/testcase/localpathstorage.go

@@ -9,9 +9,11 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var lps = "local-path-storage"
+var namespace = "local-path-storage"
 
 func TestLocalPathProvisionerStorage(cluster *shared.Cluster, applyWorkload, deleteWorkload bool) {
+	createDir(cluster)
+
 	var workloadErr error
 	if applyWorkload {
 		workloadErr = shared.ManageWorkload("apply", "local-path-provisioner.yaml")
@@ -24,22 +26,25 @@ func TestLocalPathProvisionerStorage(cluster *shared.Cluster, applyWorkload, del
 		getPodVolumeTestRunning,
 		statusRunning,
 	)
+	if err != nil {
+		logDebugData(cluster)
+	}
 	Expect(err).NotTo(HaveOccurred(), err)
 
-	_, err = shared.WriteDataPod(cluster, lps)
+	_, err = shared.WriteDataPod(cluster, namespace)
 	Expect(err).NotTo(HaveOccurred(), "error writing data to pod: %v", err)
 
 	Eventually(func(g Gomega) {
 		var res string
 		shared.LogLevel("info", "Reading data from pod")
 
-		res, err = shared.ReadDataPod(cluster, lps)
+		res, err = shared.ReadDataPod(cluster, namespace)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(res).Should(ContainSubstring("testing local path"))
 		g.Expect(err).NotTo(HaveOccurred())
 	}, "300s", "5s").Should(Succeed())
 
-	_, err = shared.ReadDataPod(cluster, lps)
+	_, err = shared.ReadDataPod(cluster, namespace)
 	if err != nil {
 		return
 	}
@@ -66,10 +71,55 @@ func readData(cluster *shared.Cluster) error {
 	delay := time.After(30 * time.Second)
 	<-delay
 
-	_, err = shared.ReadDataPod(cluster, lps)
+	_, err = shared.ReadDataPod(cluster, namespace)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+
+func createDir(cluster *shared.Cluster) {
+	shared.LogLevel("debug", "node OS: %s ", cluster.NodeOS)
+	if cluster.NodeOS == "slemicro" {
+		for _, ip := range append(cluster.ServerIPs, cluster.AgentIPs...) {
+			shared.CreateDir("/opt/data", "+w", ip)
+		}
+	}
+}
+
+// Logs the following debug data:
+// 1. pod log and describe pod output for 'helper-pod-create-pvc' pod.
+// 2. pod log and describe pod output for all pods in local-path-storage namespace
+// 3. kubectl get pv,pvc,storageclass output
+// 4. sestatus output
+// 5. grep audit logs for denied calls and log the same.
+func logDebugData(cluster *shared.Cluster) {
+	// Pod log and describe pod output for 'helper-pod-create-pvc' pod
+	shared.FindPodAndLog("helper-pod-create-pvc", "kube-system")
+
+	// Pod Log and describe pod output with namespace: local-path-storage
+	shared.LogAllPodsForNamespace(namespace)
+
+	// Log the kubectl get pv,pvc,storageclass
+	output, getErr := shared.KubectlCommand(cluster, "node", "get", "pv,pvc,storageclass", "-A")
+	if getErr != nil {
+		shared.LogLevel("error", "error getting pv,pvc and storageclass info")
+	}
+	if output != "" {
+		shared.LogLevel("debug", "pv,pvc,storageclass info:\n %s", output)
+	}
+
+	// Log sestatus output
+	cmd := "sestatus"
+	seStatusOut, statusLogErr := shared.RunCommandOnNode(cmd, cluster.ServerIPs[0])
+	if statusLogErr != nil {
+		shared.LogLevel("error", "error getting sestatus output")
+	}
+	if seStatusOut != "" {
+		shared.LogLevel("debug", "sestatus:\n %s", seStatusOut)
+	}
+
+	// Grep and Log the audit logs for denied messages
+	shared.LogGrepOutput("/var/log/audit/audit.log", "denied", cluster.ServerIPs[0])
+}

pkg/testcase/upgrademanual.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/rancher/distros-test-framework/pkg/aws"
 	"github.com/rancher/distros-test-framework/pkg/k8s"
 	"github.com/rancher/distros-test-framework/shared"
 
@@ -14,6 +15,8 @@ const (
 	server  = "server"
 	status  = "status"
 	restart = "restart"
+	stop    = "stop"
+	start   = "start"
 )
 
 // TestUpgradeClusterManual upgrades the cluster "manually".
@@ -29,10 +32,14 @@ func TestUpgradeClusterManual(cluster *shared.Cluster, k8sClient *k8s.Client, ve
 		return shared.ReturnLogError("no nodes found to upgrade")
 	}
 
+	// Initialize aws client in case reboot is needed for slemicro
+	shared.LogLevel("debug", "Testing Node OS: %s", cluster.NodeOS)
+	awsClient := getAwsClient(cluster)
+
 	// Upgrades server nodes sequentially
 	if cluster.NumServers > 0 {
 		for _, ip := range cluster.ServerIPs {
-			if err := upgradeProduct(cluster.Config.Product, server, version, ip); err != nil {
+			if err := upgradeProduct(awsClient, cluster, server, version, ip); err != nil {
 				shared.LogLevel("error", "error upgrading %s %s: %v", server, ip, err)
 				return err
 			}
@@ -42,7 +49,7 @@ func TestUpgradeClusterManual(cluster *shared.Cluster, k8sClient *k8s.Client, ve
 	// Upgrades agent nodes sequentially
 	if cluster.NumAgents > 0 {
 		for _, ip := range cluster.AgentIPs {
-			if err := upgradeProduct(cluster.Config.Product, agent, version, ip); err != nil {
+			if err := upgradeProduct(awsClient, cluster, agent, version, ip); err != nil {
 				shared.LogLevel("error", "error upgrading %s %s: %v", agent, ip, err)
 				return err
 			}
@@ -60,20 +67,37 @@ func TestUpgradeClusterManual(cluster *shared.Cluster, k8sClient *k8s.Client, ve
 	return nil
 }
 
-// upgradeProduct upgrades a node server or agent type to the specified version.
-func upgradeProduct(product, nodeType, installType, ip string) error {
-	upgradeCommand := shared.GetInstallCmd(product, installType, nodeType)
+// nodeType can be server or agent.
+// installType can be version or commit.
+func runUpgradeCommand(cluster *shared.Cluster, nodeType, installType, ip string) error {
+	upgradeCommand := shared.GetInstallCmd(cluster, installType, nodeType)
 	shared.LogLevel("info", "Upgrading %s %s: %s", ip, nodeType, upgradeCommand)
 	if _, err := shared.RunCommandOnNode(upgradeCommand, ip); err != nil {
 		shared.LogLevel("error", "error running cmd on %s %s: %v", nodeType, ip, err)
 		return err
 	}
 
-	actions := []shared.ServiceAction{
-		{Service: product, Action: restart, NodeType: nodeType, ExplicitDelay: 180},
-		{Service: product, Action: status, NodeType: nodeType, ExplicitDelay: 30},
+	return nil
+}
+
+// upgradeProduct upgrades a node server or agent type to the specified version.
+func upgradeProduct(awsClient *aws.Client, cluster *shared.Cluster, nodeType, installType, ip string) error {
+	nodeOS := cluster.NodeOS
+	product := cluster.Config.Product
+
+	err := runUpgradeCommand(cluster, nodeType, installType, ip)
+	if err != nil {
+		return err
+	}
+
+	if nodeOS == "slemicro" {
+		rebootNodeAndWait(awsClient, ip)
 	}
 
+	actions := []shared.ServiceAction{
+		{Service: product, Action: restart, NodeType: nodeType, ExplicitDelay: 60},
+		{Service: product, Action: status, NodeType: nodeType, ExplicitDelay: 120},
+	}
 	if product == "rke2" {
 		ms := shared.NewManageService(3, 30)
 		output, err := ms.ManageService(ip, actions)
@@ -88,7 +112,21 @@ func upgradeProduct(product, nodeType, installType, ip string) error {
 
 	if product == "k3s" {
 		ms := shared.NewManageService(3, 10)
-		output, err := ms.ManageService(ip, []shared.ServiceAction{actions[1]})
+		var output string
+		var err error
+		if nodeOS == "slemicro" {
+			sleActions := []shared.ServiceAction{
+				{Service: product, Action: stop, NodeType: nodeType, ExplicitDelay: 30},
+				{Service: product, Action: start, NodeType: nodeType, ExplicitDelay: 30},
+				{Service: product, Action: status, NodeType: nodeType, ExplicitDelay: 120},
+			}
+			output, err = ms.ManageService(ip, sleActions)
+		} else {
+			k3sActions := []shared.ServiceAction{
+				{Service: product, Action: status, NodeType: nodeType, ExplicitDelay: 30},
+			}
+			output, err = ms.ManageService(ip, k3sActions)
+		}
 		if output != "" {
 			Expect(output).To(ContainSubstring("active "),
 				fmt.Sprintf("error running %s service %s on %s node: %s", product, status, nodeType, ip))