Add support for setting tolerations for pods created by the daemonset (#162)

Author: dizzy57

README.md

@@ -28,6 +28,7 @@ The config values to be set are:
 | `IMAGE_PULL_SECRETS` | List of image pull secrets, in this format `pullsecret1;...` to add to pods created by the DaemonSet. Those secrets need to be in the image puller's namespace and a cluster administrator must create them.       | `""` |
 | `AFFINITY` | Affinity applied to pods created by the daemonset, in this format `'{"nodeAffinity":{ ... }}'`       | `'{}'` |
 | `KIP_IMAGE` | The image puller image to copy the `sleep` binary from | `quay.io/eclipse/kubernetes-image-puller:next` |
+| `TOLERATIONS` | Tolerations applied to pods created by the daemonset, provided in this format `'[{"operator":"Exists"}]'` | `'[]'` |
 
 ### Configuration - Helm 
 
@@ -49,6 +50,7 @@ The following values can be set:
 | `configMap.nodeSelector`         | The value of `NODE_SELECTOR` to be set in the ConfigMap      | `"{}"`                                                |
 | `configMap.imagePullSecrets` | The value of `IMAGE_PULL_SECRETS`       | `""` |
 | `configMap.affinity`         | The value of `AFFINITY` to be set in the ConfigMap      | `"{}"`                                                |
+| `configMap.tolerations`      | The value of `TOLERATIONS` to be set in the ConfigMap   | `"[]"`                                                |
 
 ### Configuration - OpenShift
 
@@ -70,6 +72,7 @@ The following values can be set:
 | `NODE_SELECTOR` | The value of `NODE_SELECTOR` to be set in the ConfigMap | `"{}"` |
 | `IMAGE_PULL_SECRETS` | The value of `IMAGE_PULL_SECRETS`       | `""` |
 | `AFFINITY` | The value of `AFFINITY` to be set in the ConfigMap | `"{}"` |
+| `TOLERATIONS` | The value of `TOLERATIONS` to be set in the ConfigMap | `"[]"` |
 
 ### Installation - Helm
 

cfg/config.go

@@ -27,6 +27,7 @@ type Config struct {
 	ImagePullSecrets  []string
 	Affinity          *corev1.Affinity
 	ImagePullerImage  string
+	Tolerations       []corev1.Toleration
 }
 
 func GetConfig() Config {
@@ -43,5 +44,6 @@ func GetConfig() Config {
 		ImagePullSecrets:  processImagePullSecretsEnvVar(),
 		Affinity:          processAffinityEnvVar(),
 		ImagePullerImage:  getEnvVarOrDefault(kipImageEnvVar, defaultImage),
+		Tolerations:       processTolerationsEnvVar(),
 	}
 }

cfg/config_test.go

@@ -40,6 +40,7 @@ func TestEnvVars(t *testing.T) {
 				ImagePullSecrets:  []string{},
 				Affinity:          &v1.Affinity{},
 				ImagePullerImage:  "quay.io/eclipse/kubernetes-image-puller:next",
+				Tolerations:       []v1.Toleration{},
 			},
 		},
 		{
@@ -52,6 +53,7 @@ func TestEnvVars(t *testing.T) {
 				"IMAGE_PULL_SECRETS":  "secret1; secret2",
 				"AFFINITY":            `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"kubernetes.io/e2e-az-name","operator":"In","values":["e2e-az1","e2e-az2"]}]}]}}}`,
 				"KIP_IMAGE":           "quay.io/my-repo/kubernetes-image-puller:next",
+				"TOLERATIONS":         `[{"effect":"NoSchedule","key":"app","operator":"Equal","value": "prod"}]`,
 			},
 			want: Config{
 				DaemonsetName: "custom-daemonset-name",
@@ -86,6 +88,14 @@ func TestEnvVars(t *testing.T) {
 					},
 				},
 				ImagePullerImage: "quay.io/my-repo/kubernetes-image-puller:next",
+				Tolerations: []v1.Toleration{
+					{
+						Key:      "app",
+						Operator: "Equal",
+						Value:    "prod",
+						Effect:   "NoSchedule",
+					},
+				},
 			},
 		},
 	}

cfg/envvars.go

@@ -36,6 +36,7 @@ const (
 	imagePullSecretsEnvVar  = "IMAGE_PULL_SECRETS"
 	affinityEnvVar          = "AFFINITY"
 	kipImageEnvVar          = "KIP_IMAGE"
+	tolerationsEnvVar       = "TOLERATIONS"
 )
 
 // Default values where applicable
@@ -52,6 +53,7 @@ const (
 	defaultImagePullSecret   = ""
 	defaultAffinity          = "{}"
 	defaultImage             = "quay.io/eclipse/kubernetes-image-puller:next"
+	defaultTolerations       = "[]"
 )
 
 func getCachingInterval() int {
@@ -128,6 +130,15 @@ func processAffinityEnvVar() *corev1.Affinity {
 	return affinity
 }
 
+func processTolerationsEnvVar() []corev1.Toleration {
+	rawTolerations := getEnvVarOrDefault(tolerationsEnvVar, defaultTolerations)
+	var tolerations []corev1.Toleration
+	if err := json.Unmarshal([]byte(rawTolerations), &tolerations); err != nil {
+		log.Fatalf("Failed to unmarshal tolerations json: %s", err)
+	}
+	return tolerations
+}
+
 func getEnvVarOrExit(envVar string) string {
 	val := os.Getenv(envVar)
 	if val == "" {

cfg/envvars_test.go

@@ -164,3 +164,64 @@ func Test_processAffinityEnvVar(t *testing.T) {
 		})
 	}
 }
+
+func TestProcessTolerationsEnvVar(t *testing.T) {
+	type testcase struct {
+		name             string
+		tolerations      string
+		isTolerationsSet bool
+		want             []v1.Toleration
+	}
+
+	tests := []testcase{
+		{
+			name:             "default tolerations, TOLERATIONS set",
+			tolerations:      "[]",
+			isTolerationsSet: true,
+			want:             []v1.Toleration{},
+		},
+		{
+			name:             "app prod, TOLERATIONS set",
+			tolerations:      `[{"effect":"NoSchedule","key":"app","operator":"Equal","value": "prod"}]`,
+			isTolerationsSet: true,
+			want: []v1.Toleration{
+				{
+					Key:      "app",
+					Operator: "Equal",
+					Value:    "prod",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+		{
+			name:             "operator exists, TOLERATIONS set",
+			tolerations:      `[{"operator":"Exists"}]`,
+			isTolerationsSet: true,
+			want: []v1.Toleration{
+				{
+					Operator: "Exists",
+				},
+			},
+		},
+		{
+			name:             "default env var, TOLERATIONS not set",
+			tolerations:      "[{\"this\": \"shouldn't be set\"}]",
+			isTolerationsSet: false,
+			want:             []v1.Toleration{},
+		},
+	}
+
+	for _, c := range tests {
+		t.Run(c.name, func(t *testing.T) {
+			defer os.Clearenv()
+			if c.isTolerationsSet {
+				os.Setenv("TOLERATIONS", c.tolerations)
+			}
+			got := processTolerationsEnvVar()
+
+			if d := cmp.Diff(c.want, got); d != "" {
+				t.Errorf("(-want, +got): %s", d)
+			}
+		})
+	}
+}

deploy/helm/templates/configmap.yaml

@@ -14,3 +14,4 @@ data:
   NODE_SELECTOR: "{{ .Values.configMap.nodeSelector }}"
   IMAGE_PULL_SECRETS: "{{ .Values.configMap.imagePullSecrets }}"
   AFFINITY: "{{ .Values.configMap.affinity }}"
+  TOLERATIONS: "{{ .Values.configMap.tolerations }}"

deploy/helm/values.yaml

@@ -18,3 +18,4 @@ configMap:
   nodeSelector: "{}"
   imagePullSecrets: ""
   affinity: "{}"
+  tolerations: "[]"

deploy/openshift/configmap.yaml

@@ -23,6 +23,7 @@ objects:
     IMAGE_PULL_SECRETS: ${IMAGE_PULL_SECRETS}
     AFFINITY: ${AFFINITY}
     KIP_IMAGE: ${KIP_IMAGE}
+    TOLERATIONS: ${TOLERATIONS}
 parameters:
 - name: IMAGES
   value: >
@@ -51,3 +52,5 @@ parameters:
   value: "{}"
 - name: KIP_IMAGE
   value: quay.io/eclipse/kubernetes-image-puller:next
+- name: TOLERATIONS
+  value: "[]"

utils/clusterutils.go

@@ -133,6 +133,7 @@ func getDaemonset(deployment *appsv1.Deployment) *appsv1.DaemonSet {
 					ImagePullSecrets: imgPullSecrets,
 					Affinity:         cfg.Affinity,
 					Volumes:          []corev1.Volume{{Name: kipVolumeName}},
+					Tolerations:      cfg.Tolerations,
 				},
 			},
 		},