Skip to content

Commit 8044ed7

Browse files
maxcao13maxcao13
committed
VPA: add InPlaceVerticalScaling feature gate to admission-controller
Adds logic to vpa admission webhook to deny requests creating VPAs with InPlaceOrRecreate update mode without enabling feature gate. Also adds admission e2e logic to wait for the vpa-webhook to be registered before starting the test. Signed-off-by: Max Cao <[email protected]>
1 parent 3825298 commit 8044ed7

File tree

13 files changed

+118
-39
lines changed

13 files changed

+118
-39
lines changed

‎vertical-pod-autoscaler/deploy/admission-controller-deployment.yaml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,3 @@ spec:
4747
- name: tls-certs
4848
secret:
4949
secretName: vpa-tls-certs
50-
---
51-
apiVersion: v1
52-
kind: Service
53-
metadata:
54-
name: vpa-webhook
55-
namespace: kube-system
56-
spec:
57-
ports:
58-
- port: 443
59-
targetPort: 8000
60-
selector:
61-
app: vpa-admission-controller

‎vertical-pod-autoscaler/deploy/admission-controller-service.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: vpa-webhook
5+
namespace: kube-system
6+
spec:
7+
ports:
8+
- port: 443
9+
targetPort: 8000
10+
selector:
11+
app: vpa-admission-controller

‎vertical-pod-autoscaler/docs/flags.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ This document is auto-generated from the flag definitions in the VPA admission-c
1212
| `--address` | ":8944" | The address to expose Prometheus metrics. |
1313
| `--alsologtostderr` | | log to standard error as well as files (no effect when -logtostderr=true) |
1414
| `--client-ca-file` | "/etc/tls-certs/caCert.pem" | Path to CA PEM file. |
15+
| `--feature-gates` | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: |
1516
| `--ignored-vpa-object-namespaces` | | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
1617
| `--kube-api-burst` | 10 | QPS burst limit when making requests to Kubernetes apiserver |
1718
| `--kube-api-qps` | 5 | QPS limit when making requests to Kubernetes apiserver |

‎vertical-pod-autoscaler/e2e/v1/actuation.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ var _ = ActuationSuiteE2eDescribe("InPlaceOrRecreate", func() {
521521
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
522522

523523
ginkgo.BeforeEach(func() {
524-
checkInPlaceOrRecreateTestsEnabled(f)
524+
checkInPlaceOrRecreateTestsEnabled(f, true, true)
525525
})
526526

527527
ginkgo.It("still applies recommendations on restart when update mode is InPlaceOrRecreate", func() {

‎vertical-pod-autoscaler/e2e/v1/admission_controller.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,19 @@ import (
3737
"github.com/onsi/gomega"
3838
)
3939

40+
const (
41+
webhookConfigName = "vpa-webhook-config"
42+
webhookName = "vpa.k8s.io"
43+
)
44+
4045
var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
4146
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
4247
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
4348

49+
ginkgo.BeforeEach(func() {
50+
waitForVpaWebhookRegistration(f)
51+
})
52+
4453
ginkgo.It("starts pods with new recommended request", func() {
4554
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
4655

@@ -909,6 +918,8 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
909918
})
910919

911920
ginkgo.It("starts pods with new recommended request with InPlaceOrRecreate mode", func() {
921+
checkInPlaceOrRecreateTestsEnabled(f, true, false)
922+
912923
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
913924

914925
ginkgo.By("Setting up a VPA CRD")
@@ -994,3 +1005,17 @@ func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment)
9941005
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when listing pods after deployment resize")
9951006
return podList
9961007
}
1008+
1009+
func waitForVpaWebhookRegistration(f *framework.Framework) {
1010+
ginkgo.By("Waiting for VPA webhook registration")
1011+
gomega.Eventually(func() bool {
1012+
webhook, err := f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), webhookConfigName, metav1.GetOptions{})
1013+
if err != nil {
1014+
return false
1015+
}
1016+
if webhook != nil && len(webhook.Webhooks) > 0 && webhook.Webhooks[0].Name == webhookName {
1017+
return true
1018+
}
1019+
return false
1020+
}, 3*time.Minute, 5*time.Second).Should(gomega.BeTrue(), "Webhook was not registered in the cluster")
1021+
}

‎vertical-pod-autoscaler/e2e/v1/common.go‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv
618618
// checkInPlaceOrRecreateTestsEnabled check for enabled feature gates in the cluster used for the
619619
// InPlaceOrRecreate VPA feature.
620620
// Use this in a "beforeEach" call before any suites that use InPlaceOrRecreate featuregate.
621-
func checkInPlaceOrRecreateTestsEnabled(f *framework.Framework) {
621+
func checkInPlaceOrRecreateTestsEnabled(f *framework.Framework, checkAdmission, checkUpdater bool) {
622622
ginkgo.By("Checking InPlacePodVerticalScaling cluster feature gate is on")
623623

624624
podList, err := f.ClientSet.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{
@@ -633,16 +633,32 @@ func checkInPlaceOrRecreateTestsEnabled(f *framework.Framework) {
633633
ginkgo.Skip("Skipping suite: InPlacePodVerticalScaling feature gate is not enabled on the cluster level")
634634
}
635635

636-
ginkgo.By("Checking InPlaceOrRecreate VPA feature gate is on")
636+
if checkUpdater {
637+
ginkgo.By("Checking InPlaceOrRecreate VPA feature gate is on for updater")
637638

638-
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
639-
gomega.Expect(err).NotTo(gomega.HaveOccurred())
640-
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
641-
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
642-
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
643-
if !anyContainsSubstring(vpaUpdaterPod.Args, string(features.InPlaceOrRecreate)) {
644-
ginkgo.Skip("Skipping suite: InPlaceOrRecreate feature gate is not enabled on the VPA level")
639+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
640+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
641+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
642+
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
643+
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
644+
if !anyContainsSubstring(vpaUpdaterPod.Args, string(features.InPlaceOrRecreate)) {
645+
ginkgo.Skip("Skipping suite: InPlaceOrRecreate feature gate is not enabled on the VPA updater")
646+
}
645647
}
648+
649+
if checkAdmission {
650+
ginkgo.By("Checking InPlaceOrRecreate VPA feature gate is on for admission controller")
651+
652+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-admission-controller", metav1.GetOptions{})
653+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
654+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
655+
vpaAdmissionPod := deploy.Spec.Template.Spec.Containers[0]
656+
gomega.Expect(vpaAdmissionPod.Name).To(gomega.Equal("admission-controller"))
657+
if !anyContainsSubstring(vpaAdmissionPod.Args, string(features.InPlaceOrRecreate)) {
658+
ginkgo.Skip("Skipping suite: InPlaceOrRecreate feature gate is not enabled on VPA admission controller")
659+
}
660+
}
661+
646662
}
647663

648664
func anyContainsSubstring(arr []string, substr string) bool {

‎vertical-pod-autoscaler/e2e/v1/updater.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ var _ = UpdaterE2eDescribe("InPlaceOrRecreate", func() {
145145
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
146146

147147
ginkgo.BeforeEach(func() {
148-
checkInPlaceOrRecreateTestsEnabled(f)
148+
checkInPlaceOrRecreateTestsEnabled(f, true, true)
149149
})
150150

151151
ginkgo.It("In-place update pods when Admission Controller status available", func() {

‎vertical-pod-autoscaler/hack/deploy-for-e2e-locally.sh‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ set -o pipefail
2020

2121
SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
2222
BASE_NAME=$(basename $0)
23-
FEATURE_GATES=${FEATURE_GATES:-""}
23+
FEATURE_GATES=${FEATURE_GATES:-""""}
2424
source "${SCRIPT_ROOT}/hack/lib/util.sh"
2525

2626
ARCH=$(kube::util::host_arch)
@@ -81,12 +81,23 @@ for i in ${COMPONENTS}; do
8181
fi
8282
if [ $i == admission-controller ] ; then
8383
(cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true)
84+
kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
8485
fi
8586
ALL_ARCHITECTURES=${ARCH} make --directory ${SCRIPT_ROOT}/pkg/${i} docker-build REGISTRY=${REGISTRY} TAG=${TAG}
8687
docker tag ${REGISTRY}/vpa-${i}-${ARCH}:${TAG} ${REGISTRY}/vpa-${i}:${TAG}
8788
kind load docker-image ${REGISTRY}/vpa-${i}:${TAG}
8889
done
8990

91+
apply_feature_gate() {
92+
component=$1
93+
if [ "${component}" == "admission-controller" ]; then
94+
kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--feature-gates='"${FEATURE_GATES}"'"}]' -o yaml -f -
95+
elif [ "${component}" == "updater" ]; then
96+
kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": ["--feature-gates='"${FEATURE_GATES}"'"]}]' -o yaml -f -
97+
else
98+
cat # passthrough
99+
fi
100+
}
90101

91102
for i in ${COMPONENTS}; do
92103
if [ $i == recommender-externalmetrics ] ; then
@@ -97,19 +108,6 @@ for i in ${COMPONENTS}; do
97108
kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/metrics-pump.yaml
98109
kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/${i}-deployment.yaml
99110
else
100-
REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f -
111+
REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | apply_feature_gate "$i" | kubectl apply -f -
101112
fi
102113
done
103-
104-
# TODO: come up with some sort of plan for feature gate enablement e2e testing purposes
105-
# only applies to updater for now
106-
107-
# --feature-gates A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
108-
# InPlaceOrRecreate=true|false (BETA - default=false)
109-
if [ -n "${FEATURE_GATES}" ]; then
110-
echo "Enabling feature gates: ${FEATURE_GATES}"
111-
kubectl -n kube-system patch deployment vpa-updater --type=json \
112-
-p='[{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": [
113-
--feature-gates='"${FEATURE_GATES}"'
114-
]}]'
115-
fi

‎vertical-pod-autoscaler/hack/deploy-for-e2e.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ gcloud auth configure-docker -q
6969
for i in ${COMPONENTS}; do
7070
if [ $i == admission-controller ] ; then
7171
(cd ${SCRIPT_ROOT}/pkg/${i} && bash ./gencerts.sh e2e || true)
72+
kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
7273
fi
7374
ALL_ARCHITECTURES=amd64 make --directory ${SCRIPT_ROOT}/pkg/${i} release
7475
done

‎vertical-pod-autoscaler/pkg/admission-controller/main.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"strings"
2525
"time"
2626

27+
"github.com/spf13/pflag"
2728
"k8s.io/client-go/informers"
2829
kube_client "k8s.io/client-go/kubernetes"
2930
kube_flag "k8s.io/component-base/cli/flag"
@@ -36,6 +37,7 @@ import (
3637
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation"
3738
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa"
3839
vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
40+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3941
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
4042
controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher"
4143
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange"
@@ -81,6 +83,7 @@ func main() {
8183
commonFlags := common.InitCommonFlags()
8284
klog.InitFlags(nil)
8385
common.InitLoggingFlags()
86+
features.MutableFeatureGate.AddFlag(pflag.CommandLine)
8487
kube_flag.InitFlags()
8588
klog.V(1).InfoS("Starting Vertical Pod Autoscaler Admission Controller", "version", common.VerticalPodAutoscalerVersion())
8689

0 commit comments

Comments
 (0)