feat: Flag to consider kubenurses on unschedulable nodes

Author: djboris9

README.md

@@ -24,6 +24,7 @@ kubenurse is configured with environment variables:
 - `KUBENURSE_EXTRA_CA`: Additional CA cert path for TLS connections
 - `KUBENURSE_NAMESPACE`: Namespace in which to look for the neighbour kubenurses
 - `KUBENURSE_NEIGHBOUR_FILTER`: A label selector to filter neighbour kubenurses
+- `KUBENURSE_ALLOW_UNSCHEDULABLE`: If this is `"true"`, path checks to neighbouring kubenurses are only made if they are running on schedulable nodes. This requires get/list/watch access to `api/v1 Node` resources
 
 Following variables are injected to the Pod by Kubernetes and should not be defined manually:
 

main.go

@@ -68,8 +68,11 @@ func main() {
 		Transport: transport,
 	}
 
+	// If we want to consider kubenurses on unschedulable nodes
+	allowUnschedulable := os.Getenv("KUBENURSE_ALLOW_UNSCHEDULABLE") == "true"
+
 	// setup checker
-	chk, err := checker.New(ctx, client, 3*time.Second)
+	chk, err := checker.New(ctx, client, 3*time.Second, allowUnschedulable)
 	if err != nil {
 		log.Fatalln(err)
 	}

pkg/checker/checker.go

@@ -14,16 +14,17 @@ import (
 
 // New configures the checker with a httpClient and a cache timeout for check
 // results. Other parameters of the Checker struct need to be configured separately.
-func New(ctx context.Context, httpClient *http.Client, cacheTTL time.Duration) (*Checker, error) {
-	discovery, err := kubediscovery.New(ctx)
+func New(ctx context.Context, httpClient *http.Client, cacheTTL time.Duration, allowUnschedulable bool) (*Checker, error) {
+	discovery, err := kubediscovery.New(ctx, allowUnschedulable)
 	if err != nil {
 		return nil, fmt.Errorf("create k8s discovery client: %w", err)
 	}
 
 	return &Checker{
-		discovery:  discovery,
-		httpClient: httpClient,
-		cacheTTL:   cacheTTL,
+		allowUnschedulable: allowUnschedulable,
+		discovery:          discovery,
+		httpClient:         httpClient,
+		cacheTTL:           cacheTTL,
 	}, nil
 }
 
@@ -110,7 +111,7 @@ func (c *Checker) MeService() (string, error) {
 func (c *Checker) checkNeighbours(nh []kubediscovery.Neighbour) {
 	for _, neighbour := range nh {
 		neighbour := neighbour // pin
-		if neighbour.NodeSchedulable {
+		if c.allowUnschedulable || neighbour.NodeSchedulable == kubediscovery.NodeSchedulable {
 			check := func() (string, error) {
 				return c.doRequest("http://" + neighbour.PodIP + ":8080/alwayshappy")
 			}

pkg/checker/types.go

@@ -20,6 +20,7 @@ type Checker struct {
 	// Neighbourhood
 	KubenurseNamespace string
 	NeighbourFilter    string
+	allowUnschedulable bool
 
 	discovery *kubediscovery.Client
 

pkg/kubediscovery/kubediscovery.go

@@ -13,22 +13,35 @@ import (
 
 // Client provides the kubediscovery client methods.
 type Client struct {
-	k8s       kubernetes.Interface
-	nodeCache *nodeCache
+	k8s                kubernetes.Interface
+	nodeCache          *nodeCache
+	allowUnschedulable bool
 }
 
+// NodeSchedulability determines if the kubernetes node is in schedulable mode
+// or not.
+type NodeSchedulability string
+
+const (
+	NodeSchedulabilityUnknown NodeSchedulability = "Unknown"
+	NodeSchedulable           NodeSchedulability = "Schedulable"
+	NodeUnschedulable         NodeSchedulability = "Unschedulable"
+)
+
 // Neighbour represents a kubenurse which should be reachable
 type Neighbour struct {
 	PodName         string
 	PodIP           string
 	HostIP          string
 	NodeName        string
-	NodeSchedulable bool
+	NodeSchedulable NodeSchedulability
 	Phase           string // Pod Phase
 }
 
 // New creates a new kubediscovery client. The context is used to stop the k8s watchers/informers.
-func New(ctx context.Context) (*Client, error) {
+// When allowUnschedulable is true, no node watcher is created and kubenurses
+// on unschedulable nodes are considered as neighbours.
+func New(ctx context.Context, allowUnschedulable bool) (*Client, error) {
 	// create in-cluster config
 	config, err := rest.InClusterConfig()
 	if err != nil {
@@ -40,14 +53,20 @@ func New(ctx context.Context) (*Client, error) {
 		return nil, fmt.Errorf("creating clientset: %w", err)
 	}
 
-	nodeCache, err := watchNodes(ctx, cliset)
-	if err != nil {
-		return nil, fmt.Errorf("starting node watcher: %w", err)
+	var nc *nodeCache
+
+	// Watch nodes only if we consider kubenurses on unschedulable nodes
+	if allowUnschedulable {
+		nc, err = watchNodes(ctx, cliset)
+		if err != nil {
+			return nil, fmt.Errorf("starting node watcher: %w", err)
+		}
 	}
 
 	return &Client{
-		k8s:       cliset,
-		nodeCache: nodeCache,
+		k8s:                cliset,
+		nodeCache:          nc,
+		allowUnschedulable: allowUnschedulable,
 	}, nil
 }
 
@@ -67,13 +86,23 @@ func (c *Client) GetNeighbours(ctx context.Context, namespace, labelSelector str
 	for idx := range pods.Items {
 		pod := pods.Items[idx]
 
+		// If we allow unschedulable kubenurses, we set the schedulability
+		// to unknown in order not to have to set up a node watcher.
+		sched := NodeSchedulabilityUnknown
+		if !c.allowUnschedulable {
+			sched = NodeUnschedulable
+			if c.nodeCache.isSchedulable(pod.Spec.NodeName) {
+				sched = NodeSchedulable
+			}
+		}
+
 		n := Neighbour{
 			PodName:         pod.Name,
 			PodIP:           pod.Status.PodIP,
 			HostIP:          pod.Status.HostIP,
 			Phase:           string(pod.Status.Phase),
 			NodeName:        pod.Spec.NodeName,
-			NodeSchedulable: c.nodeCache.isSchedulable(pod.Spec.NodeName),
+			NodeSchedulable: sched,
 		}
 		neighbours[idx] = n
 	}