Decouple readiness from controller-runtime startup (#570)

* manager, poolManager: Start controller-runtime before pool manager is ready

Currently both liveness and readiness probes are bound by the
poolManager being ready. This makes no sense, as the pod is responsive
even though the pool is being constructed.

- Decouple controller-runtime Manager startup from poolManager startup
to prevent liveness failures during pool initialization unless there is
a real connection issue.
- Add readiness probe wait mechanism to explicitly wait for the pool to
be ready.

Signed-off-by: Ram Lavi <ralavi@redhat.com>

* config: Remove publishNotReadyAddresses from webhook service

Currently the kubemacpool service is set to answer webhook calls even if
the manager is not ready.
This is not the desired behavior - webhook should not answer VMs until
pool is ready.

Removing publishNotReadyAddresses from the webhook service.

Signed-off-by: Ram Lavi <ralavi@redhat.com>

---------

Signed-off-by: Ram Lavi <ralavi@redhat.com>

Author: RamLavi

config/default/manager/manager.yaml

@@ -285,7 +285,6 @@ metadata:
   name: service
   namespace: kubemacpool-system
 spec:
-  publishNotReadyAddresses: true
   ports:
     - port: 443
       targetPort: 8000

config/release/kubemacpool.yaml

@@ -137,7 +137,6 @@ spec:
   ports:
   - port: 443
     targetPort: 8000
-  publishNotReadyAddresses: true
   selector:
     control-plane: mac-controller-manager
 ---

config/test/kubemacpool.yaml

@@ -138,7 +138,6 @@ spec:
   ports:
   - port: 443
     targetPort: 8000
-  publishNotReadyAddresses: true
   selector:
     control-plane: mac-controller-manager
 ---

pkg/manager/manager.go

@@ -133,11 +133,17 @@ func (k *KubeMacPoolManager) Run(rangeStart, rangeEnd net.HardwareAddr) error {
 			return errors.Wrap(err, "unable to register webhooks to the manager")
 		}
 
-		err = poolManager.Start()
-		if err != nil {
-			return errors.Wrap(err, "failed to start pool manager routines")
-		}
-
+		log.Info("Setting up Pool Manager")
+		go func() {
+			err := poolManager.Start()
+			if err != nil {
+				log.Error(err, "failed to start pool manager routines")
+				k.cancel()
+				return
+			}
+		}()
+
+		log.Info("Starting Runtime Manager")
 		err = k.runtimeManager.Start(ctx)
 		if err != nil {
 			log.Error(err, "unable to run the manager")

pkg/pool-manager/pool.go

@@ -26,6 +26,7 @@ import (
 	"net"
 	"reflect"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -67,6 +68,7 @@ type PoolManager struct {
 	rangeMutex       sync.RWMutex                 // mutex for range operations to support dynamic updates
 	isKubevirt       bool                         // bool if kubevirt virtualmachine crd exist in the cluster
 	waitTime         int                          // Duration in second to free macs of allocated vms that failed to start.
+	isPoolReady      atomic.Bool                  // indicates whether the pool manager has completed initialization
 }
 
 type OptMode string
@@ -131,9 +133,18 @@ func (p *PoolManager) Start() error {
 	if p.isKubevirt {
 		go p.vmWaitingCleanupLook()
 	}
+
+	log.Info("Pool Manager is ready")
+	p.isPoolReady.Store(true)
+
 	return nil
 }
 
+// IsReady returns true if the pool manager has completed initialization
+func (p *PoolManager) IsReady() bool {
+	return p.isPoolReady.Load()
+}
+
 // getManagedNamespaces pre-computes which namespaces are managed by kubemacpool for a specific webhook
 func (p *PoolManager) getManagedNamespaces(webhookName string) ([]string, error) {
 	log.V(1).Info("computing managed namespaces for initialization", "webhookName", webhookName)

pkg/webhook/webhook.go

@@ -18,6 +18,8 @@ package webhook
 
 import (
 	"crypto/tls"
+	"fmt"
+	"net/http"
 	"os"
 	"strings"
 
@@ -66,7 +68,12 @@ func AddToManager(mgr manager.Manager, poolManager *pool_manager.PoolManager) er
 		}},
 	})
 
-	s.Register("/readyz", healthz.CheckHandler{Checker: healthz.Ping})
+	s.Register("/readyz", healthz.CheckHandler{Checker: healthz.Checker(func(_ *http.Request) error {
+		if !poolManager.IsReady() {
+			return fmt.Errorf("pool manager not ready")
+		}
+		return nil
+	})})
 	s.Register("/healthz", healthz.CheckHandler{Checker: healthz.Ping})
 
 	for _, f := range AddToWebhookFuncs {