chart: Fix controller deployment liveness probe (#186)

The node plugin is daemonset living on host's network (loopback) so
kubelet can reach it via localhost. The controller is living inside
Pod's network namespace, so kubelet will not be able to reach it on
localhost. Therefore, remove `localhost` from controller's liveness
probe, and kubelet will instead use advertised Pod IP address.

Also, expose health endpoints on all addresses. This ensures that the
health endpoints are reachable on all addresses Pod is exposed on.

Added the test to confirm all containers report ready status and 0
restarts before each test. The no-restarts check might be to strict, but
let's start with that and if we detect it is causing issues, we can
relax it later.

Author: tomponline

charts/templates/lxd-csi-controller-deployment.yaml

@@ -145,7 +145,6 @@ spec:
               readOnly: true
           livenessProbe:
             httpGet:
-              host: localhost
               path: /healthz
               port: 39008
             failureThreshold: 3
@@ -249,7 +248,7 @@ spec:
           args:
             - --v=2
             - --csi-address=$(CSI_ADDRESS)
-            - --http-endpoint=127.0.0.1:39008
+            - --http-endpoint=:39008
             - --probe-timeout=3s
           env:
             - name: CSI_ADDRESS

test/e2e/e2e_debug.go

@@ -9,12 +9,35 @@ import (
 	"strings"
 	"time"
 
+	"github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes"
 )
 
+// waitContainersReady waits until all containers in the given namespace are ready.
+func waitContainersReady(ctx context.Context, client *kubernetes.Clientset, namespace string) {
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	waitReady := func(g gomega.Gomega) {
+		pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+		g.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to list pods in namespace %q", namespace)
+		g.Expect(pods.Items).NotTo(gomega.BeEmpty(), "No pods found in namespace %q", namespace)
+
+		for _, pod := range pods.Items {
+			for _, cs := range pod.Status.ContainerStatuses {
+				name := pod.Name + "/" + cs.Name
+				g.Expect(cs.RestartCount).To(gomega.BeZero(), "Container %q has restarted", name)
+				g.Expect(cs.Ready).To(gomega.BeTrue(), "Container %q is not ready", name)
+			}
+		}
+	}
+
+	gomega.Eventually(waitReady).WithContext(ctx).Should(gomega.Succeed())
+}
+
 func printControllerLogs(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, since time.Time) {
 	fmt.Printf("\n=== Controller logs ===\n")
 

test/e2e/e2e_suite_test.go

@@ -150,6 +150,10 @@ func getTestLXDStoragePool(driver string) (poolName string, cleanup func()) {
 	return poolName, cleanup
 }
 
+var _ = ginkgo.BeforeEach(func(ctx ginkgo.SpecContext) {
+	waitContainersReady(ctx, testutils.GetKubernetesClient(testutils.GetClientConfig()), "lxd-csi")
+})
+
 var _ = ginkgo.AfterEach(func() {
 	// Provide useful information when test fails.
 	rep := ginkgo.CurrentSpecReport()