rhwa nhc: add NHC & SNR sudden-loss system test

Add a test suite that runs through the following scenario.

A healthy OpenShift cluster with a MNO topology experiences the sudden
shutdown of a worker node that is running a stateful application
(a simple Pod writing to a PV). The application is evicted and
rescheduled to another healthy node in order to provide business continuity.

Assisted-by: Claude

Author: rdiscala

tests/rhwa/internal/rhwaconfig/default.yaml

@@ -1,3 +1,11 @@
 ---
 # RHWA default configurations.
-...
+
+nhc_target_worker: ""       # e.g. "openshift-worker-0.ocp.example.org"
+nhc_failover_workers: []    # e.g. ["openshift-worker-1.ocp.example.org"]
+nhc_storage_class: ""       # e.g. "ocs-external-storagecluster-ceph-rbd"
+nhc_app_image: ""           # e.g. "registry.ocp.example.org:5000/test/ubi-minimal:latest"
+nhc_target_worker_bmc:
+  address: ""               # e.g. "10.1.0.2"
+  username: ""              # e.g. "example-user"
+  password: ""              # e.g. "example-pass"

tests/rhwa/internal/rhwaconfig/rhwaconfig.go

@@ -1,10 +1,12 @@
 package rhwaconfig
 
 import (
+	"encoding/json"
 	"log"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/kelseyhightower/envconfig"
 	"github.com/rh-ecosystem-edge/eco-gotests/tests/internal/config"
@@ -16,9 +18,33 @@ const (
 	PathToDefaultRhwaParamsFile = "./default.yaml"
 )
 
+// BMCDetails holds BMC connection details for a single node.
+type BMCDetails struct {
+	Address  string `yaml:"address" json:"address"`
+	Username string `yaml:"username" json:"username"`
+	Password string `yaml:"password" json:"password"`
+}
+
+// Decode implements the envconfig.Decoder interface to parse a JSON string
+// from an environment variable into BMCDetails.
+func (b *BMCDetails) Decode(value string) error {
+	if strings.TrimSpace(value) == "" {
+		return nil
+	}
+
+	return json.Unmarshal([]byte(value), b)
+}
+
 // RHWAConfig type keeps rhwa configuration.
 type RHWAConfig struct {
 	*config.GeneralConfig
+
+	// NHC/SNR sudden-loss test configuration.
+	TargetWorker    string     `yaml:"nhc_target_worker" envconfig:"ECO_RHWA_NHC_TARGET_WORKER"`
+	FailoverWorkers []string   `yaml:"nhc_failover_workers" envconfig:"ECO_RHWA_NHC_FAILOVER_WORKERS"`
+	StorageClass    string     `yaml:"nhc_storage_class" envconfig:"ECO_RHWA_NHC_STORAGE_CLASS"`
+	AppImage        string     `yaml:"nhc_app_image" envconfig:"ECO_RHWA_NHC_APP_IMAGE"`
+	TargetWorkerBMC BMCDetails `yaml:"nhc_target_worker_bmc" envconfig:"ECO_RHWA_NHC_TARGET_WORKER_BMC"`
 }
 
 // NewRHWAConfig returns instance of RHWA config type.

tests/rhwa/nhc-operator/internal/nhcparams/const.go

@@ -3,4 +3,37 @@ package nhcparams
 const (
 	// Label represents nhc operator label that can be used for test cases selection.
 	Label = "nhc"
+
+	// LabelSuddenLoss is the label for the sudden-loss test scenario.
+	LabelSuddenLoss = "sudden-loss"
+
+	// NHCResourceName is the name of the NodeHealthCheck CR.
+	NHCResourceName = "nhc-worker-self"
+
+	// SNRDeploymentName is the name of the SNR operator deployment.
+	SNRDeploymentName = "self-node-remediation-controller-manager"
+
+	// AppNamespace is the namespace for the stateful test application.
+	AppNamespace = "stateful-app-test"
+
+	// AppName is the name of the stateful test deployment.
+	AppName = "stateful-app"
+
+	// AppLabelKey is the label key for the stateful test application.
+	AppLabelKey = "app"
+
+	// AppLabelValue is the label value for the stateful test application.
+	AppLabelValue = "stateful-app"
+
+	// AppWorkerLabel is the node label used to select worker nodes for the test app.
+	AppWorkerLabel = "node-role.kubernetes.io/appworker"
+
+	// PVCName is the name of the PersistentVolumeClaim for the test app.
+	PVCName = "app-data"
+
+	// PVCSize is the size of the PVC for the test app.
+	PVCSize = "1Gi"
+
+	// OutOfServiceTaintKey is the taint key applied by SNR to fence a node.
+	OutOfServiceTaintKey = "node.kubernetes.io/out-of-service"
 )

tests/rhwa/nhc-operator/internal/nhcparams/nhcvars.go

@@ -1,8 +1,52 @@
 package nhcparams
 
-import "github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/internal/rhwaparams"
+import (
+	"time"
+
+	"github.com/openshift-kni/k8sreporter"
+	"github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/internal/rhwaparams"
+	corev1 "k8s.io/api/core/v1"
+)
 
 var (
 	// Labels represents the range of labels that can be used for test cases selection.
 	Labels = []string{rhwaparams.Label, Label}
+
+	// OperatorDeploymentName represents NHC deployment name.
+	OperatorDeploymentName = "node-healthcheck-controller-manager"
+
+	// OperatorControllerPodLabel is how the controller pod is labeled.
+	OperatorControllerPodLabel = "node-healthcheck-operator"
+
+	// ReporterNamespacesToDump tells to the reporter from where to collect logs.
+	ReporterNamespacesToDump = map[string]string{
+		rhwaparams.RhwaOperatorNs: rhwaparams.RhwaOperatorNs,
+		AppNamespace:              AppNamespace,
+	}
+
+	// ReporterCRDsToDump tells to the reporter what CRs to dump.
+	ReporterCRDsToDump = []k8sreporter.CRData{
+		{Cr: &corev1.PodList{}},
+	}
+
+	// NodeReadyTimeout is how long to wait for a node Ready condition change.
+	NodeReadyTimeout = 2 * time.Minute
+
+	// NHCObserveTimeout is how long to wait for NHC to mark a node unhealthy.
+	NHCObserveTimeout = 3 * time.Minute
+
+	// SNRFenceTimeout is how long to wait for SNR to fence the node.
+	SNRFenceTimeout = 5 * time.Minute
+
+	// RescheduleTimeout is how long to wait for the pod to reschedule.
+	RescheduleTimeout = 5 * time.Minute
+
+	// NodeRecoveryTimeout is how long to wait for a node to become Ready after power-on.
+	NodeRecoveryTimeout = 25 * time.Minute
+
+	// PollingInterval is the default polling interval for Eventually blocks.
+	PollingInterval = 10 * time.Second
+
+	// BMCTimeout is the Redfish operation timeout.
+	BMCTimeout = 6 * time.Minute
 )

tests/rhwa/nhc-operator/nhc_suite_test.go

@@ -0,0 +1,34 @@
+package nhc
+
+import (
+	"runtime"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/rh-ecosystem-edge/eco-goinfra/pkg/reportxml"
+	"github.com/rh-ecosystem-edge/eco-gotests/tests/internal/reporter"
+	. "github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/internal/rhwainittools"
+	"github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/nhc-operator/internal/nhcparams"
+	_ "github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/nhc-operator/tests"
+)
+
+var _, currentFile, _, _ = runtime.Caller(0)
+
+func TestNHC(t *testing.T) {
+	_, reporterConfig := GinkgoConfiguration()
+	reporterConfig.JUnitReport = RHWAConfig.GetJunitReportPath(currentFile)
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "NHC", Label(nhcparams.Labels...), reporterConfig)
+}
+
+var _ = JustAfterEach(func() {
+	reporter.ReportIfFailed(
+		CurrentSpecReport(), currentFile, nhcparams.ReporterNamespacesToDump, nhcparams.ReporterCRDsToDump)
+})
+
+var _ = ReportAfterSuite("", func(report Report) {
+	reportxml.Create(
+		report, RHWAConfig.GetReportPath(), RHWAConfig.TCPrefix)
+})

tests/rhwa/nhc-operator/tests/nhc.go

@@ -0,0 +1,46 @@
+package tests
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/rh-ecosystem-edge/eco-goinfra/pkg/deployment"
+	"github.com/rh-ecosystem-edge/eco-goinfra/pkg/pod"
+
+	. "github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/internal/rhwainittools"
+	"github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/internal/rhwaparams"
+	"github.com/rh-ecosystem-edge/eco-gotests/tests/rhwa/nhc-operator/internal/nhcparams"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe(
+	"NHC Post Deployment tests",
+	Ordered,
+	ContinueOnFailure,
+	Label(nhcparams.Label), func() {
+		BeforeAll(func() {
+			By("Get NHC deployment object")
+			nhcDeployment, err := deployment.Pull(
+				APIClient, nhcparams.OperatorDeploymentName, rhwaparams.RhwaOperatorNs)
+			Expect(err).ToNot(HaveOccurred(), "Failed to get NHC deployment")
+
+			By("Verify NHC deployment is Ready")
+			Expect(nhcDeployment.IsReady(rhwaparams.DefaultTimeout)).To(BeTrue(), "NHC deployment is not Ready")
+		})
+
+		It("Verify Node Health Check Operator pod is running", func() {
+			listOptions := metav1.ListOptions{
+				LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", nhcparams.OperatorControllerPodLabel),
+			}
+			_, err := pod.WaitForAllPodsInNamespaceRunning(
+				APIClient,
+				rhwaparams.RhwaOperatorNs,
+				rhwaparams.DefaultTimeout,
+				listOptions,
+			)
+			Expect(err).ToNot(HaveOccurred(), "Pod is not ready")
+		})
+	})