Merge pull request #2229 from tnozicka/verify-no-restarts

Verify that no container in a ScyllaCluster Pod has restarted during e2e run

Author: scylla-operator-bot[bot]

test/e2e/utils/helpers.go

@@ -220,6 +220,27 @@ func GetStatefulSetsForScyllaCluster(ctx context.Context, client appv1client.App
 	return res, nil
 }
 
+func GetPodsForStatefulSet(ctx context.Context, client corev1client.CoreV1Interface, sts *appsv1.StatefulSet) (map[string]*corev1.Pod, error) {
+	selector, err := metav1.LabelSelectorAsSelector(sts.Spec.Selector)
+	if err != nil {
+		return nil, fmt.Errorf("can't convert StatefulSet %q selector: %w", naming.ObjRef(sts), err)
+	}
+
+	podList, err := client.Pods(sts.Namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: selector.String(),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("can't list Pods for StatefulSet %q: %w", naming.ObjRef(sts), err)
+	}
+
+	res := map[string]*corev1.Pod{}
+	for _, pod := range podList.Items {
+		res[pod.Name] = &pod
+	}
+
+	return res, nil
+}
+
 func GetDaemonSetsForNodeConfig(ctx context.Context, client appv1client.AppsV1Interface, nc *scyllav1alpha1.NodeConfig) ([]*appsv1.DaemonSet, error) {
 	daemonSetList, err := client.DaemonSets(naming.ScyllaOperatorNodeTuningNamespace).List(ctx, metav1.ListOptions{
 		LabelSelector: labels.Set{

test/e2e/utils/verification/scyllacluster/verify.go

@@ -2,6 +2,7 @@ package scyllacluster
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"strings"
 
@@ -59,7 +60,7 @@ func verifyPersistentVolumeClaims(ctx context.Context, coreClient corev1client.C
 	o.Expect(scPVCNames).To(o.BeEquivalentTo(expectedPvcNames))
 }
 
-func verifyStatefulset(sts *appsv1.StatefulSet, sdc *scyllav1alpha1.ScyllaDBDatacenter) {
+func verifyStatefulset(ctx context.Context, client corev1client.CoreV1Interface, sts *appsv1.StatefulSet, sdc *scyllav1alpha1.ScyllaDBDatacenter) {
 	o.Expect(sts.ObjectMeta.OwnerReferences).To(o.BeEquivalentTo(
 		[]metav1.OwnerReference{
 			{
@@ -75,8 +76,22 @@ func verifyStatefulset(sts *appsv1.StatefulSet, sdc *scyllav1alpha1.ScyllaDBData
 	o.Expect(sts.DeletionTimestamp).To(o.BeNil())
 	o.Expect(sts.Status.ObservedGeneration).To(o.Equal(sts.Generation))
 	o.Expect(sts.Spec.Replicas).NotTo(o.BeNil())
+	o.Expect(sts.Status.Replicas).To(o.Equal(*sts.Spec.Replicas))
 	o.Expect(sts.Status.ReadyReplicas).To(o.Equal(*sts.Spec.Replicas))
 	o.Expect(sts.Status.CurrentRevision).To(o.Equal(sts.Status.UpdateRevision))
+
+	// Verify pod invariants.
+	podMap, err := utils.GetPodsForStatefulSet(ctx, client, sts)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(podMap).To(o.HaveLen(int(sts.Status.Replicas)))
+
+	// No container in the Pod should be restarted.
+	for _, pod := range podMap {
+		o.Expect(pod.Status.ContainerStatuses).NotTo(o.BeEmpty())
+		for _, cs := range pod.Status.ContainerStatuses {
+			o.Expect(cs.RestartCount).To(o.BeZero(), fmt.Sprintf("container %q in pod %q should not be restarted", cs.Name, pod.Name))
+		}
+	}
 }
 
 func verifyPodDisruptionBudget(sc *scyllav1.ScyllaCluster, pdb *policyv1.PodDisruptionBudget, sdc *scyllav1alpha1.ScyllaDBDatacenter) {
@@ -279,7 +294,7 @@ func Verify(ctx context.Context, kubeClient kubernetes.Interface, scyllaClient s
 
 		s := statefulsets[r.Name]
 
-		verifyStatefulset(s, sdc)
+		verifyStatefulset(ctx, kubeClient.CoreV1(), s, sdc)
 
 		o.Expect(sc.Status.Racks[r.Name].Stale).NotTo(o.BeNil())
 		o.Expect(*sc.Status.Racks[r.Name].Stale).To(o.BeFalse())