Adding the ability to add feature gates to controller components (#1545)

Ensure PodPriority is disabled if specified in the config

Author: omar-nahhas

core/controlplane/config/config.go

@@ -357,6 +357,20 @@ func (c *Cluster) Load() error {
 	return nil
 }
 
+func (c *Cluster) ControllerFeatureGates() model.FeatureGates {
+	gates := c.Controller.NodeSettings.FeatureGates
+	//From kube 1.11 PodPriority and ExpandPersistentVolumes have become enabled by default,
+	//so making sure it is not enabled if user has explicitly set them to false
+	//https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#changelog-since-v1110
+	if !c.Experimental.Admission.Priority.Enabled {
+		gates["PodPriority"] = "false"
+	}
+	if !c.Experimental.Admission.PersistentVolumeClaimResize.Enabled {
+		gates["ExpandPersistentVolumes"] = "false"
+	}
+	return gates
+}
+
 func (c *Cluster) ConsumeDeprecatedKeys() {
 	// TODO Remove in v0.9.9-rc.1
 	if c.DeprecatedVPCID != "" {

core/controlplane/config/config_test.go

@@ -235,6 +235,28 @@ apiEndpoints:
 `,
 }
 
+var featureGates = `
+controller:
+  featureGates:
+    feature1: "true"
+    feature2: "false"
+`
+
+func TestFeatureFlags(t *testing.T) {
+	var c *Cluster
+	var err error
+	if c, err = ClusterFromBytes([]byte(singleAzConfigYaml + featureGates)); err != nil {
+		t.Errorf("Incorrect config for controller feature gates: %s\n%s", err, featureGates)
+	}
+	if c.ControllerFeatureGates().Enabled() != true {
+		t.Errorf("Incorrect config for controller feature gates: %s\n%s", err, featureGates)
+	}
+	if !(c.ControllerFeatureGates()["feature1"] == "true" &&
+		c.ControllerFeatureGates()["feature2"] == "false") {
+		t.Errorf("Incorrect config for controller feature gates: %s\n%s", err, featureGates)
+	}
+}
+
 func TestNetworkValidation(t *testing.T) {
 	for _, networkConfig := range goodNetworkingConfigs {
 		configBody := singleAzConfigYaml + networkConfig

core/controlplane/config/templates/cloud-config-controller

@@ -442,8 +442,8 @@ coreos:
         {{ else }}--cluster-dns={{.DNSServiceIP}} \
         {{ end }}--cluster-domain=cluster.local \
         --cloud-provider=aws \
-        {{if or (.Experimental.Admission.Priority.Enabled) (.Experimental.Admission.PersistentVolumeClaimResize.Enabled) -}}
-        --feature-gates=PodPriority={{.Experimental.Admission.Priority.Enabled}},ExpandPersistentVolumes={{.Experimental.Admission.PersistentVolumeClaimResize.Enabled}} \
+        {{if .ControllerFeatureGates.Enabled -}}
+        --feature-gates={{.ControllerFeatureGates.String}} \
         {{end -}}\
         {{- if .Kubelet.SystemReservedResources }}
         --system-reserved={{ .Kubelet.SystemReservedResources }} \
@@ -3485,9 +3485,9 @@ write_files:
           - --client-ca-file=/etc/kubernetes/ssl/ca.pem
           - --service-account-key-file=/etc/kubernetes/ssl/service-account-key.pem
           - --runtime-config=extensions/v1beta1/networkpolicies=true{{if .Experimental.Admission.PodSecurityPolicy.Enabled}},extensions/v1beta1/podsecuritypolicy=true{{ end }}{{if .Experimental.Admission.Initializers.Enabled}},admissionregistration.k8s.io/v1alpha1{{end}}{{if .Experimental.Admission.Priority.Enabled}},scheduling.k8s.io/v1alpha1=true{{end}}
-         {{if or (.Experimental.Admission.Priority.Enabled) (.Experimental.Admission.PersistentVolumeClaimResize.Enabled)}}
-          - --feature-gates=PodPriority={{.Experimental.Admission.Priority.Enabled}},ExpandPersistentVolumes={{.Experimental.Admission.PersistentVolumeClaimResize.Enabled}}
-         {{end}}
+          {{- if .ControllerFeatureGates.Enabled }}
+          - --feature-gates={{.ControllerFeatureGates.String}}
+          {{- end }}
           - --cloud-provider=aws
           {{ if .Addons.APIServerAggregator.Enabled -}}
           - --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem
@@ -3637,6 +3637,9 @@ write_files:
           {{ if not .Addons.MetricsServer.Enabled -}}
           - --horizontal-pod-autoscaler-use-rest-clients=false
           {{end}}
+          {{ if .ControllerFeatureGates.Enabled -}}
+          - --feature-gates={{.ControllerFeatureGates.String}}
+          {{ end -}}
           resources:
             requests:
               cpu: {{ if .Kubernetes.ControllerManager.ComputeResources.Requests.Cpu }}{{ .Kubernetes.ControllerManager.ComputeResources.Requests.Cpu }}{{ else }}100m{{ end }}
@@ -3703,9 +3706,9 @@ write_files:
           - scheduler
           - --kubeconfig=/etc/kubernetes/kubeconfig/kube-scheduler.yaml
           - --leader-elect=true
-         {{if or (.Experimental.Admission.Priority.Enabled) (.Experimental.Admission.PersistentVolumeClaimResize.Enabled)}}
-          - --feature-gates=PodPriority={{.Experimental.Admission.Priority.Enabled}},ExpandPersistentVolumes={{.Experimental.Admission.PersistentVolumeClaimResize.Enabled}}
-         {{end}}
+          {{- if .ControllerFeatureGates.Enabled }}
+          - --feature-gates={{.ControllerFeatureGates.String}}
+          {{- end }}
           resources:
             requests:
               cpu: 100m

core/nodepool/config/config.go

@@ -285,6 +285,9 @@ func (c ProvidedConfig) NodeLabels() model.NodeLabels {
 
 func (c ProvidedConfig) FeatureGates() model.FeatureGates {
 	gates := c.NodeSettings.FeatureGates
+	if gates == nil {
+		gates = model.FeatureGates{}
+	}
 	if c.Gpu.Nvidia.IsEnabledOn(c.InstanceType) {
 		gates["Accelerators"] = "true"
 	}
@@ -294,6 +297,15 @@ func (c ProvidedConfig) FeatureGates() model.FeatureGates {
 	if c.Kubelet.RotateCerts.Enabled {
 		gates["RotateKubeletClientCertificate"] = "true"
 	}
+	//From kube 1.11 PodPriority and ExpandPersistentVolumes have become enabled by default,
+	//so making sure it is not enabled if user has explicitly set them to false
+	//https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.11.md#changelog-since-v1110
+	if !c.Experimental.Admission.Priority.Enabled {
+		gates["PodPriority"] = "false"
+	}
+	if !c.Experimental.Admission.PersistentVolumeClaimResize.Enabled {
+		gates["ExpandPersistentVolumes"] = "false"
+	}
 	return gates
 }
 

test/integration/maincluster_test.go

@@ -1317,7 +1317,7 @@ experimental:
     validatingAdmissionWebhook:
       enabled: true
     persistentVolumeClaimResize:
-      enabled: true
+      enabled: false
   auditLog:
     enabled: true
     logPath: "/var/log/audit.log"
@@ -1402,7 +1402,7 @@ worker:
 								Enabled: true,
 							},
 							PersistentVolumeClaimResize: controlplane_config.PersistentVolumeClaimResize{
-								Enabled: true,
+								Enabled: false,
 							},
 						},
 						AuditLog: controlplane_config.AuditLog{
@@ -1496,7 +1496,7 @@ worker:
 				func(c root.Cluster, t *testing.T) {
 					cp := c.ControlPlane()
 					controllerUserdataS3Part := cp.UserDataController.Parts[model.USERDATA_S3].Asset.Content
-					if !strings.Contains(controllerUserdataS3Part, `--feature-gates=PodPriority=true`) {
+					if match, _ := regexp.MatchString(`--feature-gates=.*ExpandPersistentVolumes=false`, controllerUserdataS3Part); !match {
 						t.Error("missing controller feature gate: PodPriority=true")
 					}
 

test/integration/plugin_test.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"os"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -458,7 +459,7 @@ spec:
 					}
 
 					// A kube-aws plugin can activate feature gates
-					if !strings.Contains(workerUserdataS3Part, `--feature-gates="Accelerators=true"`) {
+					if match, _ := regexp.MatchString(`--feature-gates=.*Accelerators=true`, workerUserdataS3Part); !match {
 						t.Error("missing worker feature gate: Accelerators=true")
 					}