VPA: Introduce VPA feature gates; add InPlaceOrRecreate feature gate

Signed-off-by: Max Cao <[email protected]>

Author: maxcao13

vertical-pod-autoscaler/docs/flags.md

@@ -12,6 +12,7 @@ This document is auto-generated from the flag definitions in the VPA admission-c
 | `--address` | ":8944" |                         The address to expose Prometheus metrics. |
 | `--alsologtostderr` |  |                        log to standard error as well as files (no effect when -logtostderr=true) |
 | `--client-ca-file` | "/etc/tls-certs/caCert.pem" |                  Path to CA PEM file. |
+| `--feature-gates` |  |            A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: |
 | `--ignored-vpa-object-namespaces` |  |   A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
 | `--kube-api-burst` | 10 |                   QPS burst limit when making requests to Kubernetes apiserver |
 | `--kube-api-qps` | 5 |                     QPS limit when making requests to Kubernetes apiserver |
@@ -65,6 +66,7 @@ This document is auto-generated from the flag definitions in the VPA recommender
 | `--cpu-integer-post-processor-enabled` |  |                     Enable the cpu-integer recommendation post processor. The post processor will round up CPU recommendations to a whole CPU for pods which were opted in by setting an appropriate label on VPA object (experimental) |
 | `--external-metrics-cpu-metric` |  |                     ALPHA.  Metric to use with external metrics provider for CPU usage. |
 | `--external-metrics-memory-metric` |  |                  ALPHA.  Metric to use with external metrics provider for memory usage. |
+| `--feature-gates` |  |                            A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: |
 | `--history-length` | "8d" |                                  How much time back prometheus have to be queried to get historical metrics |
 | `--history-resolution` | "1h" |                              Resolution at which Prometheus is queried for historical metrics |
 | `--humanize-memory` |  |                                        Convert memory values in recommendations to the highest appropriate SI unit with up to 2 decimal places for better readability. |
@@ -135,6 +137,7 @@ This document is auto-generated from the flag definitions in the VPA updater cod
 | `--eviction-rate-burst` | 1 |                                         Burst of pods that can be evicted. |
 | `--eviction-rate-limit` |  |                                       Number of pods that can be evicted per seconds. A rate limit set to 0 or -1 will disable |
 | `--eviction-tolerance` | 0.5 |                                        Fraction of replica count that can be evicted for update, if more than one pod can be evicted. |
+| `--feature-gates` |  |                                     A set of key=value pairs that describe feature gates for alpha/experimental features. Options are: |
 | `--ignored-vpa-object-namespaces` |  |                            A comma-separated list of namespaces to ignore when searching for VPA objects. Leave empty to avoid ignoring any namespaces. These namespaces will not be cleaned by the garbage collector. |
 | `--in-recommendation-bounds-eviction-lifetime-threshold` | 12h0m0s |   Pods that live for at least that long can be evicted even if their request is within the [MinRecommended...MaxRecommended] range |
 | `--kube-api-burst` | 10 |                                            QPS burst limit when making requests to Kubernetes apiserver |

vertical-pod-autoscaler/pkg/admission-controller/main.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
 	kube_client "k8s.io/client-go/kubernetes"
 	kube_flag "k8s.io/component-base/cli/flag"
@@ -36,6 +37,7 @@ import (
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/pod/recommendation"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/admission-controller/resource/vpa"
 	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
 	controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/limitrange"
@@ -81,6 +83,7 @@ func main() {
 	commonFlags := common.InitCommonFlags()
 	klog.InitFlags(nil)
 	common.InitLoggingFlags()
+	features.MutableFeatureGate.AddFlag(pflag.CommandLine)
 	kube_flag.InitFlags()
 	klog.V(1).InfoS("Starting Vertical Pod Autoscaler Admission Controller", "version", common.VerticalPodAutoscalerVersion())
 

vertical-pod-autoscaler/pkg/features/features.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/component-base/featuregate"
+
+	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
+)
+
+const (
+	// Every feature gate should add method here following this template:
+	//
+	// // alpha: v1.X
+	// // components: admission-controller, recommender, updater
+	//
+	// MyFeature featuregate.Feature = "MyFeature".
+	//
+	// Feature gates should be listed in alphabetical, case-sensitive
+	// (upper before any lower case character) order. This reduces the risk
+	// of code conflicts because changes are more likely to be scattered
+	// across the file.
+	//
+	// In each feature gate description, you must specify "components".
+	// The feature must be enabled by the --feature-gates argument on each listed component.
+
+	// alpha: v1.4.0
+	// components: admission-controller, updater
+
+	// InPlaceOrRecreate enables the InPlaceOrRecreate update mode to be used.
+	// Requires KEP-1287 InPlacePodVerticalScaling feature-gate to be enabled on the cluster.
+	InPlaceOrRecreate featuregate.Feature = "InPlaceOrRecreate"
+)
+
+// MutableFeatureGate is a mutable, versioned, global FeatureGate.
+var MutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate()
+
+// Enabled is a helper function for MutableFeatureGate.Enabled(f)
+func Enabled(f featuregate.Feature) bool {
+	return MutableFeatureGate.Enabled(f)
+}
+
+func init() {
+	// set the emulation version to align with VPA versioning system
+	runtime.Must(MutableFeatureGate.SetEmulationVersion(version.MustParse(common.VerticalPodAutoscalerVersion())))
+	runtime.Must(MutableFeatureGate.AddVersioned(defaultVersionedFeatureGates))
+}

vertical-pod-autoscaler/pkg/features/versioned_features.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2025 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package features
+
+import (
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/component-base/featuregate"
+)
+
+// defaultVersionedFeatureGates consists of all known VPA-specific feature keys with VersionedSpecs.
+// To add a new feature, define a key for it in pkg/features/features.go and add it here. The features will be
+// available throughout Kubernetes binaries.
+
+// Entries are alphabetized.
+var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{
+	InPlaceOrRecreate: {
+		{Version: version.MustParse("1.3.0"), Default: false, PreRelease: featuregate.Alpha}, // TODO: have to downgrade to 1.3.0 for now until we bump VPA version
+	},
+}

vertical-pod-autoscaler/pkg/recommender/main.go

@@ -40,6 +40,7 @@ import (
 
 	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
 	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/checkpoint"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history"
@@ -131,6 +132,8 @@ func main() {
 	leaderElection := defaultLeaderElectionConfiguration()
 	componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine)
 
+	features.MutableFeatureGate.AddFlag(pflag.CommandLine)
+
 	kube_flag.InitFlags()
 	klog.V(1).InfoS("Vertical Pod Autoscaler Recommender", "version", common.VerticalPodAutoscalerVersion(), "recommenderName", *recommenderName)
 

vertical-pod-autoscaler/pkg/updater/main.go

@@ -37,6 +37,7 @@ import (
 
 	"k8s.io/autoscaler/vertical-pod-autoscaler/common"
 	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target"
 	controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher"
 	updater "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/updater/logic"
@@ -88,6 +89,8 @@ func main() {
 	leaderElection := defaultLeaderElectionConfiguration()
 	componentbaseoptions.BindLeaderElectionFlags(&leaderElection, pflag.CommandLine)
 
+	features.MutableFeatureGate.AddFlag(pflag.CommandLine)
+
 	kube_flag.InitFlags()
 	klog.V(1).InfoS("Vertical Pod Autoscaler Updater", "version", common.VerticalPodAutoscalerVersion())