diff --git a/interoperability/longhorn/api/client.go b/interoperability/longhorn/api/client.go new file mode 100644 index 0000000000..f9f2509b5a --- /dev/null +++ b/interoperability/longhorn/api/client.go @@ -0,0 +1,223 @@ +package api + +import ( + "context" + "fmt" + "time" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/namegenerator" + "github.com/sirupsen/logrus" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + LonghornNodeType = "longhorn.io.node" + LonghornSettingType = "longhorn.io.setting" + LonghornVolumeType = "longhorn.io.volume" +) + +// getReplicaCount determines an appropriate replica count for a Longhorn volume +// based on the number of available Longhorn nodes in the given namespace. +func getReplicaCount(client *rancher.Client, clusterID, namespace string) (int, error) { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return 0, fmt.Errorf("failed to get downstream client for replica count: %w", err) + } + + longhornNodes, err := steveClient.SteveType(LonghornNodeType).NamespacedSteveClient(namespace).List(nil) + if err != nil { + return 0, fmt.Errorf("failed to list Longhorn nodes: %w", err) + } + + nodeCount := len(longhornNodes.Data) + if nodeCount <= 0 { + return 0, fmt.Errorf("no Longhorn nodes found in namespace %s", namespace) + } + + return nodeCount, nil +} + +// CreateVolume creates a new Longhorn volume via the Rancher Steve API and returns a pointer to it +func CreateVolume(client *rancher.Client, clusterID, namespace string) (*steveV1.SteveAPIObject, error) { + volumeName := namegenerator.AppendRandomString("test-lh-vol") + + replicaCount, err := getReplicaCount(client, clusterID, namespace) + if err != nil { + return nil, err + } + + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return nil, fmt.Errorf("failed to get downstream client: %w", err) + } + + // Create volume spec + volumeSpec := map[string]interface{}{ + "type": LonghornVolumeType, + "metadata": map[string]interface{}{ + "name": volumeName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "numberOfReplicas": replicaCount, + "size": "1073741824", // 1Gi in bytes + // blockdev frontend is required for Longhorn data engine v1, which is the default storage engine + // that uses Linux kernel block devices to manage volumes + "frontend": "blockdev", + }, + } + + logrus.Infof("Creating Longhorn volume: %s with %d replicas", volumeName, replicaCount) + volume, err := steveClient.SteveType(LonghornVolumeType).Create(volumeSpec) + if err != nil { + return nil, fmt.Errorf("failed to create volume: %w", err) + } + + logrus.Infof("Successfully created volume: %s", volumeName) + + // Register cleanup function for the volume + client.Session.RegisterCleanupFunc(func() error { + logrus.Infof("Cleaning up test volume: %s", volumeName) + return DeleteVolume(client, clusterID, namespace, volumeName) + }) + + return volume, nil +} + +// ValidateVolumeActive validates that a volume is in an active/detached state and ready to use +func ValidateVolumeActive(client *rancher.Client, clusterID, namespace, volumeName string) error { + logrus.Infof("Validating volume %s is active", volumeName) + + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + err = kwait.PollUntilContextTimeout(context.TODO(), 5*time.Second, defaults.FiveMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + volumeID := fmt.Sprintf("%s/%s", namespace, volumeName) + volume, err := steveClient.SteveType(LonghornVolumeType).ByID(volumeID) + if err != nil { + // Ignore error and continue polling as volume may not be available immediately + return false, nil + } + + // Extract status from the volume + if volume.Status == nil { + return false, nil + } + + statusMap, ok := volume.Status.(map[string]interface{}) + if !ok { + return false, nil + } + + state, _ := statusMap["state"].(string) + robustness, _ := statusMap["robustness"].(string) + + logrus.Infof("Volume %s state: %s, robustness: %s", volumeName, state, robustness) + + // Volume is ready when it's in a valid state (detached or attached) with valid robustness + // Valid states: detached (ready to attach), attached (in use) + // "unknown" robustness is expected for detached volumes with no replicas scheduled + if (state == "detached" || state == "attached") && (robustness == "healthy" || robustness == "unknown") { + return true, nil + } + + return false, nil + }) + + if err != nil { + return fmt.Errorf("volume %s did not become active: %w", volumeName, err) + } + + logrus.Infof("Volume %s is active and ready to use", volumeName) + return nil +} + +// DeleteVolume deletes a Longhorn volume +func DeleteVolume(client *rancher.Client, clusterID, namespace, volumeName string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + volumeID := fmt.Sprintf("%s/%s", namespace, volumeName) + volume, err := steveClient.SteveType(LonghornVolumeType).ByID(volumeID) + if err != nil { + return fmt.Errorf("failed to get volume %s: %w", volumeName, err) + } + + logrus.Infof("Deleting volume: %s", volumeName) + err = steveClient.SteveType(LonghornVolumeType).Delete(volume) + if err != nil { + return fmt.Errorf("failed to delete volume %s: %w", volumeName, err) + } + + return nil +} + +// ValidateNodes validates that all Longhorn nodes are in a valid state +// This check is performed immediately without polling because nodes should already be +// in a ready state before Longhorn installation completes +func ValidateNodes(client *rancher.Client, clusterID, namespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + nodes, err := steveClient.SteveType(LonghornNodeType).NamespacedSteveClient(namespace).List(nil) + if err != nil { + return fmt.Errorf("failed to list nodes: %w", err) + } + + if len(nodes.Data) == 0 { + return fmt.Errorf("no Longhorn nodes found") + } + + // Validate each node has valid conditions + for _, node := range nodes.Data { + if node.Status == nil { + return fmt.Errorf("node %s has no status", node.Name) + } + } + + return nil +} + +// ValidateSettings validates that Longhorn settings are properly configured +// Checks that at least one setting has a non-nil value to ensure settings are accessible +func ValidateSettings(client *rancher.Client, clusterID, namespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + settings, err := steveClient.SteveType(LonghornSettingType).NamespacedSteveClient(namespace).List(nil) + if err != nil { + return fmt.Errorf("failed to list settings: %w", err) + } + + if len(settings.Data) == 0 { + return fmt.Errorf("no Longhorn settings found") + } + + // Validate that at least one setting has a value field + hasValidSetting := false + for _, setting := range settings.Data { + if setting.JSONResp != nil { + if _, exists := setting.JSONResp["value"]; exists { + hasValidSetting = true + break + } + } + } + + if !hasValidSetting { + return fmt.Errorf("no Longhorn settings have valid value fields") + } + + return nil +} diff --git a/validation/longhorn/README.md b/validation/longhorn/README.md index fbeae78a9e..3fa4d3e3ea 100644 --- a/validation/longhorn/README.md +++ b/validation/longhorn/README.md @@ -4,11 +4,13 @@ This directory contains tests for interoperability between Rancher and Longhorn. ## Running the tests -This package contains two test suites: -1. `TestLonghornChartTestSuite`: Tests envolving installing Longhorn through Rancher Charts. +This package contains three test suites: + +1. `TestLonghornChartTestSuite`: Tests involving installing Longhorn through Rancher Charts. 2. `TestLonghornTestSuite`: Tests that handle various other Longhorn use cases, can be run with a custom pre-installed Longhorn. +3. `TestLonghornUIAccessTestSuite`: Tests that validate Longhorn UI/API access and functionality on downstream clusters. -Additional configuration for both suites can be included in the Cattle Config file as follows: +Additional configuration for all suites can be included in the Cattle Config file as follows: ```yaml longhorn: @@ -17,3 +19,41 @@ longhorn: ``` If no additional configuration is provided, the default project name `longhorn-test` and the storage class `longhorn` are used. + +## Longhorn UI Access Test + +The `TestLonghornUIAccessTestSuite` validates Longhorn UI and API access on a downstream Rancher cluster. It performs the following checks: + +1. **Pod Validation**: Verifies all pods in the `longhorn-system` namespace are in an active/running state +2. **Service Accessibility**: Checks that the Longhorn frontend service is accessible and returns valid HTTP responses + - Supports ClusterIP (via proxy), NodePort, and LoadBalancer service types +3. **Longhorn API Validation**: + - Validates Longhorn nodes are in a valid state + - Validates Longhorn settings are properly configured + - Creates a test volume via the Longhorn API + - Verifies the volume is active through both Longhorn and Rancher APIs + - Validates the volume uses the correct Longhorn storage class + +### Test Methods + +- `TestLonghornUIAccess`: Static test that validates core functionality without user-provided configuration +- `TestLonghornUIDynamic`: Dynamic test that validates configuration based on user-provided settings in the config file + +### Running the UI Access Test + +```bash +go test -v -tags "validation" -run TestLonghornUIAccessTestSuite ./validation/longhorn/ +``` + +Or with specific test methods: + +```bash +go test -v -tags "validation" -run TestLonghornUIAccessTestSuite/TestLonghornUIAccess ./validation/longhorn/ +go test -v -tags "validation" -run TestLonghornUIAccessTestSuite/TestLonghornUIDynamic ./validation/longhorn/ +``` + +### Prerequisites + +- Longhorn must be installed on the downstream cluster (either pre-installed or installed by the test suite) +- The cluster must be accessible via Rancher +- The test requires network access to the Longhorn service in the downstream cluster diff --git a/validation/longhorn/uiaccess.go b/validation/longhorn/uiaccess.go new file mode 100644 index 0000000000..937303f128 --- /dev/null +++ b/validation/longhorn/uiaccess.go @@ -0,0 +1,202 @@ +package longhorn + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + shepherdPods "github.com/rancher/shepherd/extensions/workloads/pods" + "github.com/rancher/tests/actions/charts" + corev1 "k8s.io/api/core/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + longhornFrontendServiceName = "longhorn-frontend" +) + +// validateLonghornPods verifies that all pods in the longhorn-system namespace are in an active state +func validateLonghornPods(t *testing.T, client *rancher.Client, clusterID string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + t.Logf("Waiting for all pods in namespace %s to be running", charts.LonghornNamespace) + + // Poll until all pods are running + err = kwait.PollUntilContextTimeout(context.TODO(), 5*time.Second, defaults.FiveMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pods, err := steveClient.SteveType(shepherdPods.PodResourceSteveType).NamespacedSteveClient(charts.LonghornNamespace).List(nil) + if err != nil { + return false, nil + } + + if len(pods.Data) == 0 { + return false, nil + } + + // Check if all pods are in running state + for _, pod := range pods.Data { + if pod.State.Name != "running" { + t.Logf("Pod %s is not in running state, current state: %s", pod.Name, pod.State.Name) + return false, nil + } + } + + t.Logf("All %d pods in namespace %s are in running state", len(pods.Data), charts.LonghornNamespace) + return true, nil + }) + + if err != nil { + return fmt.Errorf("pods in namespace %s did not all reach running state: %w", charts.LonghornNamespace, err) + } + + return nil +} + +// validateLonghornService verifies that the longhorn-frontend service is accessible and returns its URL +func validateLonghornService(t *testing.T, client *rancher.Client, clusterID string) (string, error) { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return "", fmt.Errorf("failed to get downstream client: %w", err) + } + + t.Logf("Looking for service %s in namespace %s", longhornFrontendServiceName, charts.LonghornNamespace) + + // Wait for the service to be in active state + var serviceResp *steveV1.SteveAPIObject + err = kwait.PollUntilContextTimeout(context.TODO(), 5*time.Second, defaults.FiveMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + serviceID := fmt.Sprintf("%s/%s", charts.LonghornNamespace, longhornFrontendServiceName) + serviceResp, err = steveClient.SteveType(stevetypes.Service).ByID(serviceID) + if err != nil { + return false, nil + } + + if serviceResp.State.Name == "active" { + return true, nil + } + + return false, nil + }) + + if err != nil { + return "", fmt.Errorf("service %s did not become active: %w", longhornFrontendServiceName, err) + } + + t.Logf("Service %s is active", longhornFrontendServiceName) + + // Extract service information + service := &corev1.Service{} + err = steveV1.ConvertToK8sType(serviceResp.JSONResp, service) + if err != nil { + return "", fmt.Errorf("failed to convert service to k8s type: %w", err) + } + + // Construct the service URL based on the service type + var serviceURL string + switch service.Spec.Type { + case corev1.ServiceTypeClusterIP: + // For ClusterIP, use the cluster IP and port + if service.Spec.ClusterIP == "" { + return "", fmt.Errorf("service %s has no cluster IP", longhornFrontendServiceName) + } + if len(service.Spec.Ports) == 0 { + return "", fmt.Errorf("service %s has no ports defined", longhornFrontendServiceName) + } + serviceURL = fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", longhornFrontendServiceName, charts.LonghornNamespace, service.Spec.Ports[0].Port) + t.Logf("Service type is ClusterIP, URL: %s", serviceURL) + + case corev1.ServiceTypeNodePort: + // For NodePort, we need to get a node IP + if len(service.Spec.Ports) == 0 { + return "", fmt.Errorf("service %s has no ports defined", longhornFrontendServiceName) + } + nodePort := service.Spec.Ports[0].NodePort + if nodePort == 0 { + return "", fmt.Errorf("service %s has no node port defined", longhornFrontendServiceName) + } + + // Get a node IP + nodes, err := steveClient.SteveType("node").List(nil) + if err != nil { + return "", fmt.Errorf("failed to get nodes: %w", err) + } + if len(nodes.Data) == 0 { + return "", fmt.Errorf("no nodes found") + } + + node := &corev1.Node{} + err = steveV1.ConvertToK8sType(nodes.Data[0].JSONResp, node) + if err != nil { + return "", fmt.Errorf("failed to convert node to k8s type: %w", err) + } + + // Get the node's internal IP + var nodeIP string + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + nodeIP = addr.Address + break + } + } + + if nodeIP == "" { + return "", fmt.Errorf("could not find internal IP for node") + } + + serviceURL = fmt.Sprintf("http://%s:%d", nodeIP, nodePort) + t.Logf("Service type is NodePort, URL: %s", serviceURL) + + case corev1.ServiceTypeLoadBalancer: + // For LoadBalancer, use the external IP + if len(service.Spec.Ports) == 0 { + return "", fmt.Errorf("service %s has no ports defined", longhornFrontendServiceName) + } + if len(service.Status.LoadBalancer.Ingress) == 0 { + return "", fmt.Errorf("service %s has no load balancer ingress", longhornFrontendServiceName) + } + + ingress := service.Status.LoadBalancer.Ingress[0] + lbAddress := ingress.IP + if lbAddress == "" { + lbAddress = ingress.Hostname + } + serviceURL = fmt.Sprintf("http://%s:%d", lbAddress, service.Spec.Ports[0].Port) + t.Logf("Service type is LoadBalancer, URL: %s", serviceURL) + + default: + return "", fmt.Errorf("unsupported service type: %s", service.Spec.Type) + } + + return serviceURL, nil +} + +// validateLonghornStorageClassInRancher verifies that the Longhorn storage class exists and is accessible through Rancher API +func validateLonghornStorageClassInRancher(t *testing.T, client *rancher.Client, clusterID, storageClassName string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return fmt.Errorf("failed to get downstream client: %w", err) + } + + t.Logf("Looking for storage class %s in Rancher", storageClassName) + + // Get the storage class + storageClasses, err := steveClient.SteveType("storage.k8s.io.storageclass").List(nil) + if err != nil { + return fmt.Errorf("failed to list storage classes: %w", err) + } + + for _, sc := range storageClasses.Data { + if sc.Name == storageClassName { + t.Logf("Found storage class %s in Rancher", storageClassName) + return nil + } + } + + return fmt.Errorf("storage class %s not found in Rancher", storageClassName) +} diff --git a/validation/longhorn/uiaccess_test.go b/validation/longhorn/uiaccess_test.go new file mode 100644 index 0000000000..8f0b85a684 --- /dev/null +++ b/validation/longhorn/uiaccess_test.go @@ -0,0 +1,122 @@ +//go:build validation || pit.daily + +package longhorn + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/pkg/namegenerator" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + "github.com/rancher/tests/interoperability/longhorn" + longhornapi "github.com/rancher/tests/interoperability/longhorn/api" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +// LonghornUIAccessTestSuite is a test suite for validating Longhorn UI and API access on downstream clusters +type LonghornUIAccessTestSuite struct { + suite.Suite + client *rancher.Client + session *session.Session + longhornTestConfig longhorn.TestConfig + cluster *clusters.ClusterMeta + project *management.Project +} + +func (l *LonghornUIAccessTestSuite) TearDownSuite() { + l.session.Cleanup() +} + +func (l *LonghornUIAccessTestSuite) SetupSuite() { + l.session = session.NewSession() + + client, err := rancher.NewClient("", l.session) + require.NoError(l.T(), err) + l.client = client + + l.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(l.T(), err) + + l.longhornTestConfig = *longhorn.GetLonghornTestConfig() + + // Use a unique project name to avoid conflicts + projectName := namegenerator.AppendRandomString(l.longhornTestConfig.LonghornTestProject) + projectConfig := &management.Project{ + ClusterID: l.cluster.ID, + Name: projectName, + } + + l.project, err = client.Management.Project.Create(projectConfig) + require.NoError(l.T(), err) + + chart, err := shepherdCharts.GetChartStatus(l.client, l.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(l.T(), err) + + if !chart.IsAlreadyInstalled { + // Get latest versions of longhorn + latestLonghornVersion, err := l.client.Catalog.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(l.T(), err) + + payloadOpts := charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: l.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: l.cluster, + Version: latestLonghornVersion, + ProjectID: l.project.ID, + }, + } + + l.T().Logf("Installing Longhorn chart in cluster [%v] with latest version [%v] in project [%v] and namespace [%v]", l.cluster.Name, payloadOpts.Version, l.project.Name, payloadOpts.Namespace) + err = charts.InstallLonghornChart(l.client, payloadOpts, nil) + require.NoError(l.T(), err) + } +} + +func (l *LonghornUIAccessTestSuite) TestLonghornUIAccess() { + l.T().Log("Verifying all Longhorn pods are in active state") + err := validateLonghornPods(l.T(), l.client, l.cluster.ID) + require.NoError(l.T(), err) + + l.T().Log("Verifying Longhorn service is accessible") + serviceURL, err := validateLonghornService(l.T(), l.client, l.cluster.ID) + require.NoError(l.T(), err) + require.NotEmpty(l.T(), serviceURL) + + l.T().Logf("Longhorn service URL: %s", serviceURL) + + l.T().Log("Validating Longhorn nodes show valid state") + err = longhornapi.ValidateNodes(l.client, l.cluster.ID, charts.LonghornNamespace) + require.NoError(l.T(), err) + + l.T().Log("Validating Longhorn settings are properly configured") + err = longhornapi.ValidateSettings(l.client, l.cluster.ID, charts.LonghornNamespace) + require.NoError(l.T(), err) + + l.T().Log("Creating Longhorn volume through Longhorn API") + volume, err := longhornapi.CreateVolume(l.client, l.cluster.ID, charts.LonghornNamespace) + require.NoError(l.T(), err) + require.NotNil(l.T(), volume) + + volumeName := volume.Name + + l.T().Logf("Validating volume %s is active through Longhorn API", volumeName) + err = longhornapi.ValidateVolumeActive(l.client, l.cluster.ID, charts.LonghornNamespace, volumeName) + require.NoError(l.T(), err) + + l.T().Log("Verifying Longhorn storage class is accessible through Rancher API") + err = validateLonghornStorageClassInRancher(l.T(), l.client, l.cluster.ID, l.longhornTestConfig.LonghornTestStorageClass) + require.NoError(l.T(), err) +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestLonghornUIAccessTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUIAccessTestSuite)) +}