eve-k: fix node delete if replicas list encounters crd get error

Fix LonghornReplicaList to be more resilient to failure modes.
If the longhorn namespace exists pointing to an intact longhorn install,
but the replica crd is missing, return no replicas and no error.
This allows drain to be skipped and node delete or cluster delete to
continue in a time efficient process.

Fix Node Delete to be more resilient,
defer node delete to run even in all error cases.

Signed-off-by: Andrew Durbin <andrewd@zededa.com>

Author: andrewd-zededa

pkg/pillar/cmd/zedkube/drain.go

@@ -355,8 +355,23 @@ func drainAndDeleteNode(ctx *zedkube) {
 		log.Errorf("drainAndDeleteNode: can't get clientset %v", err)
 		return
 	}
-
 	nodeName := ctx.nodeName
+
+	nodeDeleteFn := func() {
+		if nodeName == "" {
+			log.Errorf("drainAndDeleteNode no nodename available for delete")
+			return
+		}
+		if err := clientset.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil {
+			log.Errorf("drainAndDeleteNode: clientset.CoreV1().Nodes().Delete failed: %v", err)
+			return
+		}
+		log.Noticef("drainAndDeleteNode: node %s drained and deleted", nodeName)
+	}
+	// For cases where the node-in-deletion is not healthy, don't
+	// leave a stale node object in the cluster, ensure its removed.
+	defer nodeDeleteFn()
+
 	node, err := clientset.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
 	if err != nil {
 		log.Errorf("drainAndDeleteNode: can't get nodes %v, for %s", err, nodeName)
@@ -414,11 +429,5 @@ func drainAndDeleteNode(ctx *zedkube) {
 			return
 		}
 	}
-
-	if err := clientset.CoreV1().Nodes().Delete(context.Background(), nodeName, metav1.DeleteOptions{}); err != nil {
-		log.Errorf("drainAndDeleteNode: clientset.CoreV1().Nodes().Delete failed: %v", err)
-		return
-	}
-	log.Noticef("drainAndDeleteNode: node %s drained and deleted", nodeName)
 	return
 }

pkg/pillar/kubeapi/longhorninfo.go

@@ -19,6 +19,7 @@ import (
 	"github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -396,6 +397,12 @@ func LonghornReplicaList(ownerNodeName string, longhornVolName string) (*lhv1bet
 		LabelSelector: strings.Join(labelSelectors, ","),
 	})
 	if err != nil {
+		// No replicas or no replicas crd is not a reason to error:
+		// eg. the server could not find the requested resource (get replicas.longhorn.io)
+		// return empty list instead of error
+		if k8serrors.IsNotFound(err) {
+			return &lhv1beta2.ReplicaList{}, nil
+		}
 		return nil, fmt.Errorf("LonghornReplicaList labelSelector:%s can't get replicas: %v", strings.Join(labelSelectors, ","), err)
 	}
 	return replicas, nil