VPA: Allow deploying InPlaceOrRecreate in local e2e and ci

Allows you to specify an env var FEATURE_GATES which adds feature gates to all vpa components during vpa-up and e2e tests.
Also allows local e2e tests to run kind with a new kind-config file which enables KEP-1287 InPlacePodVerticalScaling feature gate.
Separates the admission-controller service into a separate deploy manifest.

Signed-off-by: Max Cao <[email protected]>

Author: maxcao13

vertical-pod-autoscaler/deploy/admission-controller-deployment.yaml

@@ -47,15 +47,3 @@ spec:
         - name: tls-certs
           secret:
             secretName: vpa-tls-certs
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: vpa-webhook
-  namespace: kube-system
-spec:
-  ports:
-    - port: 443
-      targetPort: 8000
-  selector:
-    app: vpa-admission-controller

vertical-pod-autoscaler/deploy/admission-controller-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: vpa-webhook
+  namespace: kube-system
+spec:
+  ports:
+    - port: 443
+      targetPort: 8000
+  selector:
+    app: vpa-admission-controller

vertical-pod-autoscaler/docs/installation.md

@@ -138,6 +138,16 @@ To print YAML contents with all resources that would be understood by
 The output of that command won't include secret information generated by
 [pkg/admission-controller/gencerts.sh](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/pkg/admission-controller/gencerts.sh) script.
 
+### Feature gates
+
+To install VPA with feature gates, you can specify the environment variable `$FEATURE_GATES`.
+
+For example, to enable the `InPlaceOrRecreate` feature gate:
+
+```console
+FEATURE_GATES="InPlaceOrRecreate=true" ./hack/vpa-up.sh
+```
+
 ## Tear down
 
 Note that if you stop running VPA in your cluster, the resource requests

vertical-pod-autoscaler/hack/deploy-for-e2e-locally.sh

@@ -80,13 +80,13 @@ for i in ${COMPONENTS}; do
   fi
   if [ $i == admission-controller ] ; then
     (cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true)
+    kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
   fi
   ALL_ARCHITECTURES=${ARCH} make --directory ${SCRIPT_ROOT}/pkg/${i} docker-build REGISTRY=${REGISTRY} TAG=${TAG}
   docker tag ${REGISTRY}/vpa-${i}-${ARCH}:${TAG} ${REGISTRY}/vpa-${i}:${TAG}
   kind load docker-image ${REGISTRY}/vpa-${i}:${TAG}
 done
 
-
 for i in ${COMPONENTS}; do
   if [ $i == recommender-externalmetrics ] ; then
      kubectl delete namespace monitoring --ignore-not-found=true
@@ -96,6 +96,6 @@ for i in ${COMPONENTS}; do
      kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/metrics-pump.yaml
      kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/${i}-deployment.yaml
   else
-     REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh  ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f -
+    REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f -
   fi
 done

vertical-pod-autoscaler/hack/deploy-for-e2e.sh

@@ -69,6 +69,7 @@ gcloud auth configure-docker -q
 for i in ${COMPONENTS}; do
   if [ $i == admission-controller ] ; then
     (cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true)
+    kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
   fi
   ALL_ARCHITECTURES=amd64 make --directory ${SCRIPT_ROOT}/pkg/${i} release
 done

vertical-pod-autoscaler/hack/kind-config.yaml

@@ -0,0 +1,4 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+featureGates:
+  InPlacePodVerticalScaling: true

vertical-pod-autoscaler/hack/run-e2e-locally.sh

@@ -75,7 +75,7 @@ kind delete cluster -n kind -q
 
 echo "Creating KIND cluster 'kind'"
 KIND_VERSION="kindest/node:v1.32.0"
-if ! kind create cluster --image=${KIND_VERSION}; then
+if ! kind create cluster --image=${KIND_VERSION} --config ${SCRIPT_ROOT}/hack/kind-config.yaml; then
     echo "Failed to create KIND cluster. Exiting. Make sure kind version is updated."
     echo "Available versions: https://github.com/kubernetes-sigs/kind/releases"
     exit 1

vertical-pod-autoscaler/hack/vpa-process-yaml.sh

@@ -18,14 +18,34 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
-
 function print_help {
   echo "ERROR! Usage: vpa-process-yaml.sh <YAML files>+"
   echo "Script will output content of YAML files separated with YAML document"
   echo "separator and substituting REGISTRY and TAG for pod images"
 }
 
+# Requires input from stdin, otherwise hangs. Checks for "admission-controller", "updater", or "recommender", and
+# applies the respective kubectl patch command to add the feature gates specified in the FEATURE_GATES environment variable.
+# e.g. cat file.yaml | apply_feature_gate
+function apply_feature_gate() {
+  local input=""
+  while IFS= read -r line; do
+      input+="$line"$'\n'
+  done
+
+  if [ -n "${FEATURE_GATES}" ]; then
+    if echo "$input" | grep -q "admission-controller"; then
+      echo "$input" | kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--feature-gates='"${FEATURE_GATES}"'"}]' -o yaml -f -
+    elif echo "$input" | grep -q "updater" || echo "$input" | grep -q "recommender"; then
+      echo "$input" | kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": ["--feature-gates='"${FEATURE_GATES}"'"]}]' -o yaml -f -
+    else
+      echo "$input"
+    fi
+  else
+    echo "$input"
+  fi
+}
+
 if [ $# -eq 0 ]; then
   print_help
   exit 1
@@ -36,6 +56,7 @@ DEFAULT_TAG="1.3.0"
 
 REGISTRY_TO_APPLY=${REGISTRY-$DEFAULT_REGISTRY}
 TAG_TO_APPLY=${TAG-$DEFAULT_TAG}
+FEATURE_GATES=${FEATURE_GATES:-""}
 
 if [ "${REGISTRY_TO_APPLY}" != "${DEFAULT_REGISTRY}" ]; then
   (>&2 echo "WARNING! Using image repository from REGISTRY env variable (${REGISTRY_TO_APPLY}) instead of ${DEFAULT_REGISTRY}.")
@@ -46,7 +67,6 @@ if [ "${TAG_TO_APPLY}" != "${DEFAULT_TAG}" ]; then
 fi
 
 for i in $*; do
-  sed -e "s,${DEFAULT_REGISTRY}/\([a-z-]*\):.*,${REGISTRY_TO_APPLY}/\1:${TAG_TO_APPLY}," $i
-  echo ""
+  sed -e "s,${DEFAULT_REGISTRY}/\([a-z-]*\):.*,${REGISTRY_TO_APPLY}/\1:${TAG_TO_APPLY}," $i | apply_feature_gate
   echo "---"
 done

vertical-pod-autoscaler/hack/vpa-process-yamls.sh

@@ -66,9 +66,11 @@ for i in $COMPONENTS; do
     if [[ ${ACTION} == create || ${ACTION} == apply ]] ; then
       # Allow gencerts to fail silently if certs already exist
       (bash ${SCRIPT_ROOT}/pkg/admission-controller/gencerts.sh || true)
+      kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
     elif [ ${ACTION} == delete ] ; then
       (bash ${SCRIPT_ROOT}/pkg/admission-controller/rmcerts.sh || true)
       (bash ${SCRIPT_ROOT}/pkg/admission-controller/delete-webhook.sh || true)
+      kubectl delete -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
     fi
   fi
   if [[ ${ACTION} == print ]]; then