e2e: check PVC health in workload Health()

Fail when application PVCs are Lost or being deleted, and treat
Pending as retryable so wait loops can recover during deploy.

Signed-off-by: raaizik <132667934+raaizik@users.noreply.github.com>

Author: raaizik

e2e/deployers/retry.go

@@ -4,6 +4,7 @@
 package deployers
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -64,6 +65,10 @@ func WaitWorkloadHealth(ctx types.TestContext, cluster *types.Cluster) error {
 			return nil
 		}
 
+		if errors.Is(err, util.ErrUnrecoverable) {
+			return err
+		}
+
 		if err := util.Sleep(ctx.Context(), util.RetryInterval); err != nil {
 			return fmt.Errorf("workload \"%s/%s\" is not healthy in cluster %q: %w",
 				ctx.AppNamespace(), w.GetAppName(), cluster.Name, err)

e2e/util/errors.go

@@ -0,0 +1,9 @@
+// SPDX-FileCopyrightText: The RamenDR authors
+// SPDX-License-Identifier: Apache-2.0
+
+package util
+
+import "errors"
+
+// ErrUnrecoverable marks a health check failure that should not be retried.
+var ErrUnrecoverable = errors.New("unrecoverable error")

e2e/workloads/deploy.go

@@ -158,7 +158,7 @@ func (w Deployment) Health(ctx types.TestContext, cluster *types.Cluster) error
 			condition.Reason, deploymentMinimumReplicasAvailable, cluster.Name)
 	}
 
-	return nil
+	return checkPVCHealth(ctx, cluster, ctx.AppNamespace(), deploymentPVCName)
 }
 
 // Status returns the deployment status across managed clusters.

e2e/workloads/pvc.go

@@ -0,0 +1,53 @@
+// SPDX-FileCopyrightText: The RamenDR authors
+// SPDX-License-Identifier: Apache-2.0
+
+package workloads
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
+	"github.com/ramendr/ramen/e2e/types"
+	"github.com/ramendr/ramen/e2e/util"
+)
+
+// checkPVCHealth verifies the application PVC is Bound and not being deleted.
+// Bound returns nil. Pending and NotFound are retryable. Lost or a set
+// deletionTimestamp wrap util.ErrUnrecoverable so wait loops fail immediately.
+func checkPVCHealth(
+	ctx types.TestContext,
+	cluster *types.Cluster,
+	namespace, pvcName string,
+) error {
+	pvc, err := getPVC(ctx, cluster, namespace, pvcName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("pvc \"%s/%s\" not found in cluster %q",
+				namespace, pvcName, cluster.Name)
+		}
+
+		return fmt.Errorf("pvc \"%s/%s\" not available in cluster %q: %w",
+			namespace, pvcName, cluster.Name, err)
+	}
+
+	if pvc.DeletionTimestamp != nil {
+		return fmt.Errorf("pvc \"%s/%s\" is being deleted in cluster %q: %w",
+			namespace, pvcName, cluster.Name, util.ErrUnrecoverable)
+	}
+
+	switch pvc.Status.Phase {
+	case corev1.ClaimBound:
+		return nil
+	case corev1.ClaimPending:
+		return fmt.Errorf("pvc \"%s/%s\" phase is %q, expected %q in cluster %q",
+			namespace, pvcName, pvc.Status.Phase, corev1.ClaimBound, cluster.Name)
+	case corev1.ClaimLost:
+		return fmt.Errorf("pvc \"%s/%s\" phase is %q in cluster %q: %w",
+			namespace, pvcName, pvc.Status.Phase, cluster.Name, util.ErrUnrecoverable)
+	default:
+		return fmt.Errorf("pvc \"%s/%s\" phase is %q, expected %q in cluster %q",
+			namespace, pvcName, pvc.Status.Phase, corev1.ClaimBound, cluster.Name)
+	}
+}

e2e/workloads/vm.go

@@ -124,7 +124,7 @@ func (w *VM) Health(ctx types.TestContext, cluster *types.Cluster) error {
 			condition.Status, cluster.Name, condition.Message)
 	}
 
-	return nil
+	return checkPVCHealth(ctx, cluster, ctx.AppNamespace(), vmPVCName)
 }
 
 // Status returns the VM workload deployment status across managed clusters.