Skip to content

Commit 348b5ee

Browse files
maxcao13maxcao13
committed
VPA: add InPlaceVerticalScaling feature gate to admission-controller
Adds logic to vpa admission webhook to deny requests creating VPAs with InPlaceOrRecreate update mode without enabling feature gate. Also adds admission e2e logic to wait for the vpa-webhook to be registered before starting the test. Signed-off-by: Max Cao <[email protected]>
1 parent b8648eb commit 348b5ee

File tree

12 files changed

+118
-37
lines changed

12 files changed

+118
-37
lines changed

‎vertical-pod-autoscaler/deploy/admission-controller-deployment.yaml

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,3 @@ spec:
4747
- name: tls-certs
4848
secret:
4949
secretName: vpa-tls-certs
50-
---
51-
apiVersion: v1
52-
kind: Service
53-
metadata:
54-
name: vpa-webhook
55-
namespace: kube-system
56-
spec:
57-
ports:
58-
- port: 443
59-
targetPort: 8000
60-
selector:
61-
app: vpa-admission-controller

‎vertical-pod-autoscaler/deploy/admission-controller-service.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: vpa-webhook
5+
namespace: kube-system
6+
spec:
7+
ports:
8+
- port: 443
9+
targetPort: 8000
10+
selector:
11+
app: vpa-admission-controller

‎vertical-pod-autoscaler/docs/flags.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ This document is auto-generated from the flag definitions in the VPA admission-c
1212
| `--address` | ":8944" | The address to expose Prometheus metrics. |
1313
| `--alsologtostderr` | | log to standard error as well as files (no effect when -logtostderr=true) |
1414
| `--client-ca-file` | "/etc/tls-certs/caCert.pem" | Path to CA PEM file. |
15+
| `--feature-gates` | | A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: |
1516
| `--ignored-vpa-object-namespaces` | | A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
1617
| `--kube-api-burst` | 10 | QPS burst limit when making requests to Kubernetes apiserver |
1718
| `--kube-api-qps` | 5 | QPS limit when making requests to Kubernetes apiserver |

‎vertical-pod-autoscaler/e2e/v1/actuation.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ var _ = ActuationSuiteE2eDescribe("InPlaceVerticalScaling", func() {
521521
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
522522

523523
ginkgo.BeforeEach(func() {
524-
checkInPlaceVerticalScalingTestsEnabled(f)
524+
checkInPlaceVerticalScalingTestsEnabled(f, true, true)
525525
})
526526

527527
ginkgo.It("still applies recommendations on restart when update mode is InPlaceOrRecreate", func() {

‎vertical-pod-autoscaler/e2e/v1/admission_controller.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,19 @@ import (
3737
"github.com/onsi/gomega"
3838
)
3939

40+
const (
41+
webhookConfigName = "vpa-webhook-config"
42+
webhookName = "vpa.k8s.io"
43+
)
44+
4045
var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
4146
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
4247
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
4348

49+
ginkgo.BeforeEach(func() {
50+
waitForVpaWebhookRegistration(f)
51+
})
52+
4453
ginkgo.It("starts pods with new recommended request", func() {
4554
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
4655

@@ -909,6 +918,8 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
909918
})
910919

911920
ginkgo.It("starts pods with new recommended request with InPlaceOrRecreate mode", func() {
921+
checkInPlaceVerticalScalingTestsEnabled(f, true, false)
922+
912923
d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
913924

914925
ginkgo.By("Setting up a VPA CRD")
@@ -994,3 +1005,17 @@ func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment)
9941005
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "when listing pods after deployment resize")
9951006
return podList
9961007
}
1008+
1009+
func waitForVpaWebhookRegistration(f *framework.Framework) {
1010+
ginkgo.By("Waiting for VPA webhook registration")
1011+
gomega.Eventually(func() bool {
1012+
webhook, err := f.ClientSet.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), webhookConfigName, metav1.GetOptions{})
1013+
if err != nil {
1014+
return false
1015+
}
1016+
if webhook != nil && len(webhook.Webhooks) > 0 && webhook.Webhooks[0].Name == webhookName {
1017+
return true
1018+
}
1019+
return false
1020+
}, 3*time.Minute, 5*time.Second).Should(gomega.BeTrue(), "Webhook was not registered in the cluster")
1021+
}

‎vertical-pod-autoscaler/e2e/v1/common.go

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv
618618
// checkInPlaceVerticalScalingTestsEnabled check for enabled feature gates in the cluster used for the
619619
// InPlaceVerticalScaling VPA feature.
620620
// Use this in a "beforeEach" call before any suites that use InPlaceVerticalScaling featuregate.
621-
func checkInPlaceVerticalScalingTestsEnabled(f *framework.Framework) {
621+
func checkInPlaceVerticalScalingTestsEnabled(f *framework.Framework, checkAdmission, checkUpdater bool) {
622622
ginkgo.By("Checking InPlacePodVerticalScaling cluster feature gate is on")
623623

624624
podList, err := f.ClientSet.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{
@@ -633,16 +633,32 @@ func checkInPlaceVerticalScalingTestsEnabled(f *framework.Framework) {
633633
ginkgo.Skip("Skipping suite: InPlacePodVerticalScaling feature gate is not enabled on the cluster level")
634634
}
635635

636-
ginkgo.By("Checking InPlaceVerticalScaling VPA feature gate is on")
636+
if checkUpdater {
637+
ginkgo.By("Checking InPlaceVerticalScaling VPA feature gate is on for updater")
637638

638-
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
639-
gomega.Expect(err).NotTo(gomega.HaveOccurred())
640-
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
641-
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
642-
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
643-
if !anyContainsSubstring(vpaUpdaterPod.Args, string(features.InPlaceVerticalScaling)) {
644-
ginkgo.Skip("Skipping suite: InPlaceVerticalScaling feature gate is not enabled on the VPA level")
639+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
640+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
641+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
642+
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
643+
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
644+
if !anyContainsSubstring(vpaUpdaterPod.Args, string(features.InPlaceVerticalScaling)) {
645+
ginkgo.Skip("Skipping suite: InPlaceVerticalScaling feature gate is not enabled on the VPA updater")
646+
}
645647
}
648+
649+
if checkAdmission {
650+
ginkgo.By("Checking InPlaceVerticalScaling VPA feature gate is on for admission controller")
651+
652+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-admission-controller", metav1.GetOptions{})
653+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
654+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
655+
vpaAdmissionPod := deploy.Spec.Template.Spec.Containers[0]
656+
gomega.Expect(vpaAdmissionPod.Name).To(gomega.Equal("admission-controller"))
657+
if !anyContainsSubstring(vpaAdmissionPod.Args, string(features.InPlaceVerticalScaling)) {
658+
ginkgo.Skip("Skipping suite: InPlaceVerticalScaling feature gate is not enabled on VPA admission controller")
659+
}
660+
}
661+
646662
}
647663

648664
func anyContainsSubstring(arr []string, substr string) bool {

‎vertical-pod-autoscaler/e2e/v1/updater.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ var _ = UpdaterE2eDescribe("InPlaceVerticalScaling", func() {
145145
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
146146

147147
ginkgo.BeforeEach(func() {
148-
checkInPlaceVerticalScalingTestsEnabled(f)
148+
checkInPlaceVerticalScalingTestsEnabled(f, true, true)
149149
})
150150

151151
ginkgo.It("In-place update pods when Admission Controller status available", func() {

‎vertical-pod-autoscaler/hack/deploy-for-e2e-locally.sh

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ set -o pipefail
2020

2121
SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
2222
BASE_NAME=$(basename $0)
23-
FEATURE_GATES=${FEATURE_GATES:-""}
23+
FEATURE_GATES=${FEATURE_GATES:-""""}
2424
source "${SCRIPT_ROOT}/hack/lib/util.sh"
2525

2626
ARCH=$(kube::util::host_arch)
@@ -87,6 +87,16 @@ for i in ${COMPONENTS}; do
8787
kind load docker-image ${REGISTRY}/vpa-${i}:${TAG}
8888
done
8989

90+
apply_feature_gate() {
91+
component=$1
92+
if [ "${component}" == "admission-controller" ]; then
93+
kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--feature-gates='"${FEATURE_GATES}"'"}]' -o yaml -f -
94+
elif [ "${component}" == "updater" ]; then
95+
kubectl patch --type=json --local -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": ["--feature-gates='"${FEATURE_GATES}"'"]}]' -o yaml -f -
96+
else
97+
cat # passthrough
98+
fi
99+
}
90100

91101
for i in ${COMPONENTS}; do
92102
if [ $i == recommender-externalmetrics ] ; then
@@ -97,19 +107,11 @@ for i in ${COMPONENTS}; do
97107
kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/metrics-pump.yaml
98108
kubectl apply -f ${SCRIPT_ROOT}/hack/e2e/${i}-deployment.yaml
99109
else
100-
REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | kubectl apply -f -
110+
REGISTRY=${REGISTRY} TAG=${TAG} ${SCRIPT_ROOT}/hack/vpa-process-yaml.sh ${SCRIPT_ROOT}/deploy/${i}-deployment.yaml | apply_feature_gate "$i" | kubectl apply -f -
111+
if [ $i == admission-controller ] ; then
112+
kubectl apply -f ${SCRIPT_ROOT}/deploy/admission-controller-service.yaml
113+
fi
101114
fi
102115
done
103116

104-
# TODO: come up with some sort of plan for feature gate enablement e2e testing purposes
105-
# only applies to updater for now
106-
107-
# --feature-gates A set of key=value pairs that describe feature gates for alpha/experimental features. Options are:
108-
# InPlaceVerticalScaling=true|false (BETA - default=false)
109-
if [ -n "${FEATURE_GATES}" ]; then
110-
echo "Enabling feature gates: ${FEATURE_GATES}"
111-
kubectl -n kube-system patch deployment vpa-updater --type=json \
112-
-p='[{"op": "add", "path": "/spec/template/spec/containers/0/args", "value": [
113-
--feature-gates='"${FEATURE_GATES}"'
114-
]}]'
115-
fi
117+
exit

‎vertical-pod-autoscaler/pkg/admission-controller/main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"strings"
2525
"time"
2626

27+
"github.com/spf13/pflag"
2728
"k8s.io/client-go/informers"
2829
kube_client "k8s.io/client-go/kubernetes"
2930
kube_flag "k8s.io/component-base/cli/flag"
@@ -36,6 +37,7 @@ import (
3637
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation"
3738
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa"
3839
vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
40+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3941
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
4042
controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher"
4143
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange"
@@ -81,6 +83,7 @@ func main() {
8183
commonFlags := common.InitCommonFlags()
8284
klog.InitFlags(nil)
8385
common.InitLoggingFlags()
86+
features.MutableFeatureGate.AddFlag(pflag.CommandLine)
8487
kube_flag.InitFlags()
8588
klog.V(1).InfoS("Starting Vertical Pod Autoscaler Admission Controller", "version", common.VerticalPodAutoscalerVersion())
8689

‎vertical-pod-autoscaler/pkg/admission-controller/resource/vpa/handler.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030

3131
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource"
3232
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
33+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3334
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/metrics/admission"
3435
)
3536

@@ -122,6 +123,9 @@ func ValidateVPA(vpa *vpa_types.VerticalPodAutoscaler, isCreate bool) error {
122123
if _, found := possibleUpdateModes[*mode]; !found {
123124
return fmt.Errorf("unexpected UpdateMode value %s", *mode)
124125
}
126+
if (*mode == vpa_types.UpdateModeInPlaceOrRecreate) && !features.Enabled(features.InPlaceVerticalScaling) {
127+
return fmt.Errorf("in order to use UpdateMode %s, you must enable feature gate %s in the admission and updater args", vpa_types.UpdateModeInPlaceOrRecreate, features.InPlaceVerticalScaling)
128+
}
125129

126130
if minReplicas := vpa.Spec.UpdatePolicy.MinReplicas; minReplicas != nil && *minReplicas <= 0 {
127131
return fmt.Errorf("MinReplicas has to be positive, got %v", *minReplicas)

0 commit comments

Comments
 (0)