From b66b44621e78886d7350e9f6334baf1a5a3e9554 Mon Sep 17 00:00:00 2001 From: Rachel Gregory Date: Tue, 11 Feb 2025 14:31:35 -0800 Subject: [PATCH 01/39] Update skewer version on master branch --- cluster-autoscaler/cloudprovider/azure/azure_client.go | 2 +- cluster-autoscaler/go.mod | 4 ++-- cluster-autoscaler/go.sum | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index fbc39a62ed2..b875f32168d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index f5227ad3e1f..4841210eea1 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -16,7 +16,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/skewer v0.0.14 + github.com/Azure/skewer v0.0.19 github.com/aws/aws-sdk-go v1.44.241 github.com/cenkalti/backoff/v4 v4.3.0 github.com/digitalocean/godo v1.27.0 @@ -52,6 +52,7 @@ require ( k8s.io/cloud-provider-gcp/providers v0.28.2 k8s.io/component-base v0.33.0-alpha.0 k8s.io/component-helpers v0.33.0-alpha.0 + k8s.io/dynamic-resource-allocation v0.0.0 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.33.0-alpha.0 k8s.io/kubernetes v1.33.0-alpha.0 @@ -206,7 +207,6 @@ require ( k8s.io/cri-api v0.33.0-alpha.0 // indirect k8s.io/cri-client v0.0.0 // indirect k8s.io/csi-translation-lib v0.27.0 // indirect - k8s.io/dynamic-resource-allocation v0.0.0 // indirect k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/kms v0.33.0-alpha.0 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 540100deaa3..472a76bd271 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -77,6 +77,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/skewer v0.0.14 h1:0mzUJhspECkajYyynYsOCp//E2PSnYXrgP45bcskqfQ= github.com/Azure/skewer v0.0.14/go.mod h1:6WTecuPyfGtuvS8Mh4JYWuHhO4kcWycGfsUBB+XTFG4= +github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= +github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= From 72665b3d1ca03431a147f973d5d33109fce2675d Mon Sep 17 00:00:00 2001 From: Rachel Gregory Date: Thu, 20 Feb 2025 14:57:51 -0800 Subject: [PATCH 02/39] Undo previous changes made by go mod vendor This reverts commit b66b44621e78886d7350e9f6334baf1a5a3e9554. --- cluster-autoscaler/cloudprovider/azure/azure_client.go | 2 +- cluster-autoscaler/go.mod | 4 ++-- cluster-autoscaler/go.sum | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index b875f32168d..fbc39a62ed2 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 4841210eea1..f5227ad3e1f 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -16,7 +16,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/skewer v0.0.19 + github.com/Azure/skewer v0.0.14 github.com/aws/aws-sdk-go v1.44.241 github.com/cenkalti/backoff/v4 v4.3.0 github.com/digitalocean/godo v1.27.0 @@ -52,7 +52,6 @@ require ( k8s.io/cloud-provider-gcp/providers v0.28.2 k8s.io/component-base v0.33.0-alpha.0 k8s.io/component-helpers v0.33.0-alpha.0 - k8s.io/dynamic-resource-allocation v0.0.0 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.33.0-alpha.0 k8s.io/kubernetes v1.33.0-alpha.0 @@ -207,6 +206,7 @@ require ( k8s.io/cri-api v0.33.0-alpha.0 // indirect k8s.io/cri-client v0.0.0 // indirect k8s.io/csi-translation-lib v0.27.0 // indirect + k8s.io/dynamic-resource-allocation v0.0.0 // indirect k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/kms v0.33.0-alpha.0 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 472a76bd271..540100deaa3 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -77,8 +77,6 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/skewer v0.0.14 h1:0mzUJhspECkajYyynYsOCp//E2PSnYXrgP45bcskqfQ= github.com/Azure/skewer v0.0.14/go.mod h1:6WTecuPyfGtuvS8Mh4JYWuHhO4kcWycGfsUBB+XTFG4= -github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= -github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= From ed621282b5826c9dc7ad37d97dd5bf7425b0a52d Mon Sep 17 00:00:00 2001 From: Rachel Gregory Date: Thu, 20 Feb 2025 15:12:40 -0800 Subject: [PATCH 03/39] Update only skewer with go get dep@ver --- cluster-autoscaler/cloudprovider/azure/azure_client.go | 2 +- cluster-autoscaler/go.mod | 2 +- cluster-autoscaler/go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index fbc39a62ed2..b875f32168d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index f5227ad3e1f..ab3671e73d0 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -16,7 +16,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/skewer v0.0.14 + github.com/Azure/skewer v0.0.19 github.com/aws/aws-sdk-go v1.44.241 github.com/cenkalti/backoff/v4 v4.3.0 github.com/digitalocean/godo v1.27.0 diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 540100deaa3..472a76bd271 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -77,6 +77,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/skewer v0.0.14 h1:0mzUJhspECkajYyynYsOCp//E2PSnYXrgP45bcskqfQ= github.com/Azure/skewer v0.0.14/go.mod h1:6WTecuPyfGtuvS8Mh4JYWuHhO4kcWycGfsUBB+XTFG4= +github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= +github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= From c76a7712489040f5486534bf65fc2f00ec773424 Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Mon, 24 Feb 2025 14:03:56 -0500 Subject: [PATCH 04/39] Support CPU Startup Boost in VPA --- .../enhancements/cpu-startup-boost/README.md | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md diff --git a/vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md new file mode 100644 index 00000000000..5331e6c5285 --- /dev/null +++ b/vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md @@ -0,0 +1,148 @@ +# AEP-: CPU Startup Boost + + +- [AEP-: CPU Startup Boost](#aep--cpu-startup-boost) + - [Summary](#summary) + - [Goals](#goals) + - [Non-Goals](#non-goals) + - [Proposal](#proposal) + - [Design Details](#design-details) + - [Workflow](#workflow) + - [API Changes](#api-changes) + - [Priority of `StartupBoost`](#priority-of-startupboost) + - [CPU startup boost and VPA autoscaling modes](#cpu-startup-boost-and-vpa-autoscaling-modes) + - [Validation](#validation) + - [Startup Boost and Readiness/Startup Probes](#startup-boost-and-readinessstartup-probes) + - [Failed in-place downsizes](#failed-in-place-downsizes) + - [Test Plan](#test-plan) + - [Implementation History](#implementation-history) + + +## Summary + +Long application start time is a known problem for more traditional workloads +running in containerized applications, especially Java workloads. This delay can +negatively impact the user experience and overall application performance. One +potential solution is to provide additional CPU resources to pods during their +startup phase, but this can lead to waste if the extra CPU resources are not +set back to their original values after the pods are ready. + +This proposal allows VPA to boost the CPU request and limit of containers during +the pod startup and to scale the CPU resources back down when the pod is `Ready`, +leveraging the [in-place pod resize Kubernetes feature](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). + + +### Goals + +* Allow VPA to boost the CPU request and limit of a pod's containers when the pod during the pod startup (from creation time until it becomes `Ready`.) +* Allow VPA to scale pods down to the original CPU resource values, [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources), as soon as their [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) condition is true. + +### Non-Goals + +* Allow VPA to boost CPU resources of pods that are already [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions). +* Allow VPA to boost CPU resources during startup of workloads that have not configured a [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). +* Allow VPA to boost memory resources. + +## Proposal + +To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190) with a `StartupBoost.CPU.Factor` field, that will instruct VPA the factor by which to multiply the initial resource request and limit of the containers' targeted by the VPA object. + +This AEP also proposes to extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L230-L235) with a new `StartupBoostOnly` mode to allow users to only enable the startup boost feature and not vanilla VPA altogether. + +## Design Details + +### Workflow + +1. The user first configures the CPU startup boost factor on their VPA object + +1. When a pod targeted by that VPA is created, the kube-apiserver invokes the +VPA Admission Controller + +1. The VPA Admission Controller modifies the pod's containers CPU request and +limits to align with its `boostPolicy`, if specified, during the +pod creation. + * VPA Admission Controller will not modify the CPU resources of a pod if + it does not configure a Readiness or a Startup probe. + +1. The VPA Updater monitors pods targeted by the VPA object and when the pod +condition is `Ready`, it scales down the CPU resources to the appropriate +non-boosted value: `max(existing VPA recommendation for that container, minimum configured value)`. + * The scale down is applied [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). + +### API Changes + +To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190) with a `StartupBoost.CPU.Factor` field, that will instruct VPA the factor by which to multiply the initial resource request and limit of the containers' targeted by the VPA object. + +This AEP also proposes to extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L230-L235) with a new `StartupBoostOnly` mode to allow users to only enable the startup boost feature and not vanilla VPA altogether. + + +#### Priority of `StartupBoost` + +The new `StartupBoost` field will take precedence over the rest of the container resource policy configurations. Functioning independently from all other fields in [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190), except for [`ContainerName`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191-L194) and [`Mode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L195-L197). This means that a container's CPU request/limit can be boosted during startup beyond [`MaxAllowed`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L202-L205), for example, or it will be able to be boosted even if +CPU is explicitly excluded from [`ControlledResources`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L207-L211). + +#### CPU startup boost and VPA autoscaling modes + +Initially, CPU startup boost will only be able to be used if the container or VPA scaling mode is [`InPlaceOrRecreate`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L61), since it does not make sense to use this feature with disruptive modes of VPA + +Here's an example of the VPA CRD, incorporating CPU boosting (only shows relevant sections): + +```yaml +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: example-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: example + updatePolicy: + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + - containerName: 'boosted-container' + mode: 'StartupBoostOnly' + startupBoost: + cpu: + factor: 2.0 +``` + +### Validation + +* We'll also validate that the `startupBoost` configuration is valid when VPA objects are created/updated: + * The VPA autoscaling mode must be `InPlaceOrRecreate` + * The boost factor is >= 1 + +### Startup Boost and Readiness/Startup Probes + +The CPU Startup Boost functionality will boost the CPU requests of containers +until the pod has a `Ready` condition. Therefore, in this version of the AEP, +workloads must be configured with a Readiness or a Startup probe to be able to +utilize this feature. + +### Failed in-place downsizes + +The VPA Updater **will not** evict a pod to actuate a startup CPU boost recommendation if it attempted to apply the recommendation in place and it failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA updater will consider that the update failed). This is to avoid an eviction loop: + +1. A pod is created and has its CPU resources boosted +1. The pod is ready. VPA Updater tries to downscale the pod in-place and it fails. +1. VPA Updater evicts the pod. Logic flow goes back to (1). + +## Test Plan + +* Add tests to verify the VPA fields validation + +We will add the following scenarios to our e2e tests: + +* CPU Startup Boost recommendation is applied to pod controlled by VPA until it becomes `Ready`. Then, the pod is scaled back down in-place. + * Boost is applied to all containers of a pod. + * Boost is applied to a subset of containers. +* CPU Startup Boost will not be applied if a pod is not configured with a Readiness or a Startup probe. +* Pod should not be evicted if the in-place update fails when scaling the pod back down. + + +## Implementation History + +* 2025-02-24: Initial version. + From 4b98746f36f1a9661198e4ad43d527d7fe3d92fc Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Mon, 24 Feb 2025 14:20:00 -0500 Subject: [PATCH 05/39] Add AEP ID See https://github.com/kubernetes/autoscaler/issues/7862 --- .../{cpu-startup-boost => 7862-cpu-startup-boost}/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename vertical-pod-autoscaler/enhancements/{cpu-startup-boost => 7862-cpu-startup-boost}/README.md (98%) diff --git a/vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md similarity index 98% rename from vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md rename to vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md index 5331e6c5285..cd2267f4594 100644 --- a/vertical-pod-autoscaler/enhancements/cpu-startup-boost/README.md +++ b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md @@ -1,7 +1,7 @@ -# AEP-: CPU Startup Boost +# AEP-7862: CPU Startup Boost -- [AEP-: CPU Startup Boost](#aep--cpu-startup-boost) +- [AEP-7862: CPU Startup Boost](#aep-7862-cpu-startup-boost) - [Summary](#summary) - [Goals](#goals) - [Non-Goals](#non-goals) From 2aba67154cd4f117da4702b60a10c38c0651e659 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov Date: Tue, 4 Mar 2025 08:47:02 +0200 Subject: [PATCH 06/39] Fixes histograms becoming empty after loaded from checkpoints --- .../pkg/recommender/util/histogram.go | 1 + .../pkg/recommender/util/histogram_test.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go index cd0d360d0f4..08eeca55c5f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go @@ -277,6 +277,7 @@ func (h *histogram) LoadFromCheckpoint(checkpoint *vpa_types.HistogramCheckpoint h.bucketWeight[bucket] += float64(weight) * ratio } h.totalWeight += checkpoint.TotalWeight + h.updateMinAndMaxBucket() return nil } diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go index 09e2fb1706f..259c0646389 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go @@ -265,6 +265,21 @@ func TestHistogramLoadFromCheckpointReturnsErrorOnNilInput(t *testing.T) { assert.Error(t, err) } +func TestHistogramIsNotEmptyAfterSavingAndLoadingCheckpointsWithBoundaryValues(t *testing.T) { + histogram := NewHistogram(testHistogramOptions) + histogram.AddSample(1, weightEpsilon, anyTime) + histogram.AddSample(2, (float64(MaxCheckpointWeight)*weightEpsilon - weightEpsilon), anyTime) + assert.False(t, histogram.IsEmpty()) + + checkpoint, err := histogram.SaveToChekpoint() + assert.NoError(t, err) + + newHistogram := NewHistogram(testHistogramOptions) + err = newHistogram.LoadFromCheckpoint(checkpoint) + assert.NoError(t, err) + assert.False(t, newHistogram.IsEmpty()) +} + func areUnique(values ...interface{}) bool { dict := make(map[interface{}]bool) for i, v := range values { From 18ed03650207239bd682cdf59a0907a2008aa81a Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Wed, 5 Mar 2025 17:57:00 +0100 Subject: [PATCH 07/39] Convert ClusterState to interface --- .../checkpoint/checkpoint_writer.go | 10 +- .../checkpoint/checkpoint_writer_test.go | 4 +- .../pkg/recommender/input/cluster_feeder.go | 22 +-- .../recommender/input/cluster_feeder_test.go | 65 +++++-- .../input/history/history_provider.go | 2 +- .../input/metrics/metrics_source.go | 6 +- .../model/aggregate_container_state.go | 4 +- .../model/aggregate_container_state_test.go | 4 +- .../pkg/recommender/model/cluster.go | 184 +++++++++++------- .../pkg/recommender/model/cluster_test.go | 38 ++-- .../pkg/recommender/routines/recommender.go | 14 +- 11 files changed, 216 insertions(+), 137 deletions(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go index db3b5d70ff6..ca97df21d62 100644 --- a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go +++ b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go @@ -42,11 +42,11 @@ type CheckpointWriter interface { type checkpointWriter struct { vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter - cluster *model.ClusterState + cluster model.ClusterState } // NewCheckpointWriter returns new instance of a CheckpointWriter -func NewCheckpointWriter(cluster *model.ClusterState, vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter) CheckpointWriter { +func NewCheckpointWriter(cluster model.ClusterState, vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter) CheckpointWriter { return &checkpointWriter{ vpaCheckpointClient: vpaCheckpointClient, cluster: cluster, @@ -77,7 +77,7 @@ func getVpasToCheckpoint(clusterVpas map[model.VpaID]*model.Vpa) []*model.Vpa { } func (writer *checkpointWriter) StoreCheckpoints(ctx context.Context, now time.Time, minCheckpoints int) error { - vpas := getVpasToCheckpoint(writer.cluster.Vpas) + vpas := getVpasToCheckpoint(writer.cluster.VPAs()) for _, vpa := range vpas { // Draining ctx.Done() channel. ctx.Err() will be checked if timeout occurred, but minCheckpoints have @@ -123,13 +123,13 @@ func (writer *checkpointWriter) StoreCheckpoints(ctx context.Context, now time.T // Build the AggregateContainerState for the purpose of the checkpoint. This is an aggregation of state of all // containers that belong to pods matched by the VPA. // Note however that we exclude the most recent memory peak for each container (see below). -func buildAggregateContainerStateMap(vpa *model.Vpa, cluster *model.ClusterState, now time.Time) map[string]*model.AggregateContainerState { +func buildAggregateContainerStateMap(vpa *model.Vpa, cluster model.ClusterState, now time.Time) map[string]*model.AggregateContainerState { aggregateContainerStateMap := vpa.AggregateStateByContainerName() // Note: the memory peak from the current (ongoing) aggregation interval is not included in the // checkpoint to avoid having multiple peaks in the same interval after the state is restored from // the checkpoint. Therefore we are extracting the current peak from all containers. // TODO: Avoid the nested loop over all containers for each VPA. - for _, pod := range cluster.Pods { + for _, pod := range cluster.Pods() { for containerName, container := range pod.Containers { aggregateKey := cluster.MakeAggregateStateKey(pod, containerName) if vpa.UsesAggregation(aggregateKey) { diff --git a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go index 6828be9ad46..c9dc14fc79f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go @@ -54,7 +54,7 @@ var ( const testGcPeriod = time.Minute -func addVpa(t *testing.T, cluster *model.ClusterState, vpaID model.VpaID, selector string) *model.Vpa { +func addVpa(t *testing.T, cluster model.ClusterState, vpaID model.VpaID, selector string) *model.Vpa { var apiObject vpa_types.VerticalPodAutoscaler apiObject.Namespace = vpaID.Namespace apiObject.Name = vpaID.VpaName @@ -64,7 +64,7 @@ func addVpa(t *testing.T, cluster *model.ClusterState, vpaID model.VpaID, select if err != nil { t.Fatalf("AddOrUpdateVpa() failed: %v", err) } - return cluster.Vpas[vpaID] + return cluster.VPAs()[vpaID] } func TestMergeContainerStateForCheckpointDropsRecentMemoryPeak(t *testing.T) { diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index 587581dd194..1cd82a9a765 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -55,7 +55,7 @@ const ( DefaultRecommenderName = "default" ) -// ClusterStateFeeder can update state of ClusterState object. +// ClusterStateFeeder can update state of clusterState object. type ClusterStateFeeder interface { // InitFromHistoryProvider loads historical pod spec into clusterState. InitFromHistoryProvider(historyProvider history.HistoryProvider) @@ -78,7 +78,7 @@ type ClusterStateFeeder interface { // ClusterStateFeederFactory makes instances of ClusterStateFeeder. type ClusterStateFeederFactory struct { - ClusterState *model.ClusterState + ClusterState model.ClusterState KubeClient kube_client.Interface MetricsClient metrics.MetricsClient VpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter @@ -202,7 +202,7 @@ type clusterStateFeeder struct { oomChan <-chan oom.OomInfo vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter vpaLister vpa_lister.VerticalPodAutoscalerLister - clusterState *model.ClusterState + clusterState model.ClusterState selectorFetcher target.VpaTargetSelectorFetcher memorySaveMode bool controllerFetcher controllerfetcher.ControllerFetcher @@ -244,7 +244,7 @@ func (feeder *clusterStateFeeder) InitFromHistoryProvider(historyProvider histor func (feeder *clusterStateFeeder) setVpaCheckpoint(checkpoint *vpa_types.VerticalPodAutoscalerCheckpoint) error { vpaID := model.VpaID{Namespace: checkpoint.Namespace, VpaName: checkpoint.Spec.VPAObjectName} - vpa, exists := feeder.clusterState.Vpas[vpaID] + vpa, exists := feeder.clusterState.VPAs()[vpaID] if !exists { return fmt.Errorf("cannot load checkpoint to missing VPA object %s/%s", vpaID.Namespace, vpaID.VpaName) } @@ -263,7 +263,7 @@ func (feeder *clusterStateFeeder) InitFromCheckpoints() { feeder.LoadVPAs(context.TODO()) namespaces := make(map[string]bool) - for _, v := range feeder.clusterState.Vpas { + for _, v := range feeder.clusterState.VPAs() { namespaces[v.ID.Namespace] = true } @@ -431,15 +431,15 @@ func (feeder *clusterStateFeeder) LoadVPAs(ctx context.Context) { for _, condition := range conditions { if condition.delete { - delete(feeder.clusterState.Vpas[vpaID].Conditions, condition.conditionType) + delete(feeder.clusterState.VPAs()[vpaID].Conditions, condition.conditionType) } else { - feeder.clusterState.Vpas[vpaID].Conditions.Set(condition.conditionType, true, "", condition.message) + feeder.clusterState.VPAs()[vpaID].Conditions.Set(condition.conditionType, true, "", condition.message) } } } } // Delete non-existent VPAs from the model. - for vpaID := range feeder.clusterState.Vpas { + for vpaID := range feeder.clusterState.VPAs() { if _, exists := vpaKeys[vpaID]; !exists { klog.V(3).InfoS("Deleting VPA", "vpa", klog.KRef(vpaID.Namespace, vpaID.VpaName)) if err := feeder.clusterState.DeleteVpa(vpaID); err != nil { @@ -447,7 +447,7 @@ func (feeder *clusterStateFeeder) LoadVPAs(ctx context.Context) { } } } - feeder.clusterState.ObservedVpas = vpaCRDs + feeder.clusterState.SetObservedVPAs(vpaCRDs) } // LoadPods loads pod into the cluster state. @@ -460,7 +460,7 @@ func (feeder *clusterStateFeeder) LoadPods() { for _, spec := range podSpecs { pods[spec.ID] = spec } - for key := range feeder.clusterState.Pods { + for key := range feeder.clusterState.Pods() { if _, exists := pods[key]; !exists { klog.V(3).InfoS("Deleting Pod", "pod", klog.KRef(key.Namespace, key.PodName)) feeder.clusterState.DeletePod(key) @@ -518,7 +518,7 @@ Loop: } func (feeder *clusterStateFeeder) matchesVPA(pod *spec.BasicPodSpec) bool { - for vpaKey, vpa := range feeder.clusterState.Vpas { + for vpaKey, vpa := range feeder.clusterState.VPAs() { podLabels := labels.Set(pod.PodLabels) if vpaKey.Namespace == pod.ID.Namespace && vpa.PodSelector.Matches(podLabels) { return true diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index af50c4388b5..dcc311671f8 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -77,7 +77,46 @@ const ( testGcPeriod = time.Minute ) -func TestLoadPods(t *testing.T) { +// NewClusterState returns a new clusterState with no pods. +func NewFakeClusterState(vpas map[model.VpaID]*model.Vpa, pods map[model.PodID]*model.PodState) *fakeClusterState { + return &fakeClusterState{ + stubbedVPAs: vpas, + stubbedPods: pods, + addedSamples: make(map[model.ContainerID][]*model.ContainerUsageSampleWithKey), + } +} + +type fakeClusterState struct { + model.ClusterState + addedPods []model.PodID + addedSamples map[model.ContainerID][]*model.ContainerUsageSampleWithKey + stubbedVPAs map[model.VpaID]*model.Vpa + stubbedPods map[model.PodID]*model.PodState +} + +func (cs *fakeClusterState) AddSample(sample *model.ContainerUsageSampleWithKey) error { + samplesForContainer := cs.addedSamples[sample.Container] + cs.addedSamples[sample.Container] = append(samplesForContainer, sample) + return nil +} + +func (cs *fakeClusterState) AddOrUpdatePod(podID model.PodID, _ labels.Set, _ v1.PodPhase) { + cs.addedPods = append(cs.addedPods, podID) +} + +func (cs *fakeClusterState) Pods() map[model.PodID]*model.PodState { + return cs.stubbedPods +} + +func (cs *fakeClusterState) VPAs() map[model.VpaID]*model.Vpa { + return cs.stubbedVPAs +} + +func (cs *fakeClusterState) StateMapSize() int { + return 0 +} + +func TestLoadVPAs(t *testing.T) { type testCase struct { name string @@ -329,11 +368,11 @@ func TestLoadPods(t *testing.T) { } if !tc.expectedVpaFetch { - assert.NotContains(t, clusterState.Vpas, vpaID) + assert.NotContains(t, clusterState.VPAs(), vpaID) return } - assert.Contains(t, clusterState.Vpas, vpaID) - storedVpa := clusterState.Vpas[vpaID] + assert.Contains(t, clusterState.VPAs(), vpaID) + storedVpa := clusterState.VPAs()[vpaID] if tc.expectedSelector != nil { assert.NotNil(t, storedVpa.PodSelector) assert.Equal(t, tc.expectedSelector.String(), storedVpa.PodSelector.String()) @@ -427,14 +466,14 @@ func TestClusterStateFeeder_LoadPods(t *testing.T) { }, } { t.Run(tc.Name, func(t *testing.T) { - clusterState := model.NewClusterState(testGcPeriod) + vpas := make(map[model.VpaID]*model.Vpa) for i, selector := range tc.VPALabelSelectors { vpaLabel, err := labels.Parse(selector) assert.NoError(t, err) - clusterState.Vpas = map[model.VpaID]*model.Vpa{ - {VpaName: fmt.Sprintf("test-vpa-%d", i), Namespace: "default"}: {PodSelector: vpaLabel}, - } + key := model.VpaID{VpaName: fmt.Sprintf("test-vpa-%d", i), Namespace: "default"} + vpas[key] = &model.Vpa{PodSelector: vpaLabel} } + clusterState := NewFakeClusterState(vpas, nil) feeder := clusterStateFeeder{ specClient: makeTestSpecClient(tc.PodLabels), @@ -443,7 +482,9 @@ func TestClusterStateFeeder_LoadPods(t *testing.T) { } feeder.LoadPods() - assert.Len(t, feeder.clusterState.Pods, tc.TrackedPods, "number of pods is not %d", tc.TrackedPods) + assert.Len(t, clusterState.addedPods, tc.TrackedPods, "number of pods is not %d", tc.TrackedPods) + + clusterState = NewFakeClusterState(vpas, nil) feeder = clusterStateFeeder{ specClient: makeTestSpecClient(tc.PodLabels), @@ -506,10 +547,10 @@ func TestClusterStateFeeder_InitFromHistoryProvider(t *testing.T) { clusterState: clusterState, } feeder.InitFromHistoryProvider(&provider) - if !assert.Contains(t, feeder.clusterState.Pods, pod1) { + if !assert.Contains(t, feeder.clusterState.Pods(), pod1) { return } - pod1State := feeder.clusterState.Pods[pod1] + pod1State := feeder.clusterState.Pods()[pod1] if !assert.Contains(t, pod1State.Containers, containerCpu) { return } @@ -700,7 +741,7 @@ func TestCanCleanupCheckpoints(t *testing.T) { coreClient: client.CoreV1(), vpaLister: vpaLister, vpaCheckpointClient: checkpointClient, - clusterState: &model.ClusterState{}, + clusterState: model.NewClusterState(testGcPeriod), recommenderName: "default", } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go b/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go index 0db773fc52c..968765b9171 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go @@ -75,7 +75,7 @@ func newEmptyHistory() *PodHistory { // HistoryProvider gives history of all pods in a cluster. // TODO(schylek): this interface imposes how history is represented which doesn't work well with checkpoints. -// Consider refactoring to passing ClusterState and create history provider working with checkpoints. +// Consider refactoring to passing clusterState and create history provider working with checkpoints. type HistoryProvider interface { GetClusterHistory() (map[model.PodID]*PodHistory, error) } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go b/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go index 7f8905e3f73..f86dcc17d8f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go @@ -58,7 +58,7 @@ func (s podMetricsSource) List(ctx context.Context, namespace string, opts v1.Li type externalMetricsClient struct { externalClient external_metrics.ExternalMetricsClient options ExternalClientOptions - clusterState *model.ClusterState + clusterState model.ClusterState } // ExternalClientOptions specifies parameters for using an External Metrics Client. @@ -69,7 +69,7 @@ type ExternalClientOptions struct { } // NewExternalClient returns a Source for an External Metrics Client. -func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options ExternalClientOptions) PodMetricsLister { +func NewExternalClient(c *rest.Config, clusterState model.ClusterState, options ExternalClientOptions) PodMetricsLister { extClient, err := external_metrics.NewForConfig(c) if err != nil { klog.ErrorS(err, "Failed initializing external metrics client") @@ -85,7 +85,7 @@ func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options func (s *externalMetricsClient) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { result := v1beta1.PodMetricsList{} - for _, vpa := range s.clusterState.Vpas { + for _, vpa := range s.clusterState.VPAs() { if vpa.PodCount == 0 { continue } diff --git a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go index c45025bab5d..c18e860f4ff 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go @@ -307,12 +307,12 @@ func AggregateStateByContainerName(aggregateContainerStateMap aggregateContainer // present in the cluster state. type ContainerStateAggregatorProxy struct { containerID ContainerID - cluster *ClusterState + cluster *clusterState } // NewContainerStateAggregatorProxy creates a ContainerStateAggregatorProxy // pointing to the cluster state. -func NewContainerStateAggregatorProxy(cluster *ClusterState, containerID ContainerID) ContainerStateAggregator { +func NewContainerStateAggregatorProxy(cluster *clusterState, containerID ContainerID) ContainerStateAggregator { return &ContainerStateAggregatorProxy{containerID, cluster} } diff --git a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go index 0886bd5db9b..fd0e2f79a7f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go @@ -38,7 +38,7 @@ var ( } ) -func addTestCPUSample(cluster *ClusterState, container ContainerID, cpuCores float64) error { +func addTestCPUSample(cluster ClusterState, container ContainerID, cpuCores float64) error { sample := ContainerUsageSampleWithKey{ Container: container, ContainerUsageSample: ContainerUsageSample{ @@ -50,7 +50,7 @@ func addTestCPUSample(cluster *ClusterState, container ContainerID, cpuCores flo return cluster.AddSample(&sample) } -func addTestMemorySample(cluster *ClusterState, container ContainerID, memoryBytes float64) error { +func addTestMemorySample(cluster ClusterState, container ContainerID, memoryBytes float64) error { sample := ContainerUsageSampleWithKey{ Container: container, ContainerUsageSample: ContainerUsageSample{ diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go index 8fd2fd8f030..c6c6ca674af 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go @@ -35,22 +35,17 @@ const ( RecommendationMissingMaxDuration = 30 * time.Minute ) -// ClusterState holds all runtime information about the cluster required for the -// VPA operations, i.e. configuration of resources (pods, containers, -// VPA objects), aggregated utilization of compute resources (CPU, memory) and -// events (container OOMs). -// All input to the VPA Recommender algorithm lives in this structure. -type ClusterState struct { +type clusterState struct { // Pods in the cluster. - Pods map[PodID]*PodState + pods map[PodID]*PodState // VPA objects in the cluster. - Vpas map[VpaID]*Vpa + vpas map[VpaID]*Vpa // VPA objects in the cluster that have no recommendation mapped to the first // time we've noticed the recommendation missing or last time we logged // a warning about it. - EmptyVPAs map[VpaID]time.Time + emptyVPAs map[VpaID]time.Time // Observed VPAs. Used to check if there are updates needed. - ObservedVpas []*vpa_types.VerticalPodAutoscaler + observedVpas []*vpa_types.VerticalPodAutoscaler // All container aggregations where the usage samples are stored. aggregateStateMap aggregateContainerStatesMap @@ -63,7 +58,7 @@ type ClusterState struct { } // StateMapSize is the number of pods being tracked by the VPA -func (cluster *ClusterState) StateMapSize() int { +func (cluster *clusterState) StateMapSize() int { return len(cluster.aggregateStateMap) } @@ -97,12 +92,12 @@ type PodState struct { Phase apiv1.PodPhase } -// NewClusterState returns a new ClusterState with no pods. -func NewClusterState(gcInterval time.Duration) *ClusterState { - return &ClusterState{ - Pods: make(map[PodID]*PodState), - Vpas: make(map[VpaID]*Vpa), - EmptyVPAs: make(map[VpaID]time.Time), +// NewClusterState returns a new clusterState with no pods. +func NewClusterState(gcInterval time.Duration) *clusterState { + return &clusterState{ + pods: make(map[PodID]*PodState), + vpas: make(map[VpaID]*Vpa), + emptyVPAs: make(map[VpaID]time.Time), aggregateStateMap: make(aggregateContainerStatesMap), labelSetMap: make(labelSetMap), lastAggregateContainerStateGC: time.Unix(0, 0), @@ -122,11 +117,11 @@ type ContainerUsageSampleWithKey struct { // the Cluster object. // If the labels of the pod have changed, it updates the links between the containers // and the aggregations. -func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) { - pod, podExists := cluster.Pods[podID] +func (cluster *clusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) { + pod, podExists := cluster.pods[podID] if !podExists { pod = newPod(podID) - cluster.Pods[podID] = pod + cluster.pods[podID] = pod } newlabelSetKey := cluster.getLabelSetKey(newLabels) @@ -149,8 +144,8 @@ func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, p // addPodToItsVpa increases the count of Pods associated with a VPA object. // Does a scan similar to findOrCreateAggregateContainerState so could be optimized if needed. -func (cluster *ClusterState) addPodToItsVpa(pod *PodState) { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) addPodToItsVpa(pod *PodState) { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { vpa.PodCount++ } @@ -158,8 +153,8 @@ func (cluster *ClusterState) addPodToItsVpa(pod *PodState) { } // removePodFromItsVpa decreases the count of Pods associated with a VPA object. -func (cluster *ClusterState) removePodFromItsVpa(pod *PodState) { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) removePodFromItsVpa(pod *PodState) { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { vpa.PodCount-- } @@ -168,8 +163,8 @@ func (cluster *ClusterState) removePodFromItsVpa(pod *PodState) { // GetContainer returns the ContainerState object for a given ContainerID or // null if it's not present in the model. -func (cluster *ClusterState) GetContainer(containerID ContainerID) *ContainerState { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) GetContainer(containerID ContainerID) *ContainerState { + pod, podExists := cluster.pods[containerID.PodID] if podExists { container, containerExists := pod.Containers[containerID.ContainerName] if containerExists { @@ -180,20 +175,20 @@ func (cluster *ClusterState) GetContainer(containerID ContainerID) *ContainerSta } // DeletePod removes an existing pod from the cluster. -func (cluster *ClusterState) DeletePod(podID PodID) { - pod, found := cluster.Pods[podID] +func (cluster *clusterState) DeletePod(podID PodID) { + pod, found := cluster.pods[podID] if found { cluster.removePodFromItsVpa(pod) } - delete(cluster.Pods, podID) + delete(cluster.pods, podID) } // AddOrUpdateContainer creates a new container with the given ContainerID and -// adds it to the parent pod in the ClusterState object, if not yet present. -// Requires the pod to be added to the ClusterState first. Otherwise an error is +// adds it to the parent pod in the clusterState object, if not yet present. +// Requires the pod to be added to the clusterState first. Otherwise an error is // returned. -func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID, request Resources) error { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) AddOrUpdateContainer(containerID ContainerID, request Resources) error { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { return NewKeyError(containerID.PodID) } @@ -207,11 +202,11 @@ func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID, reque return nil } -// AddSample adds a new usage sample to the proper container in the ClusterState +// AddSample adds a new usage sample to the proper container in the clusterState // object. Requires the container as well as the parent pod to be added to the -// ClusterState first. Otherwise an error is returned. -func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) error { - pod, podExists := cluster.Pods[sample.Container.PodID] +// clusterState first. Otherwise an error is returned. +func (cluster *clusterState) AddSample(sample *ContainerUsageSampleWithKey) error { + pod, podExists := cluster.pods[sample.Container.PodID] if !podExists { return NewKeyError(sample.Container.PodID) } @@ -226,8 +221,8 @@ func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) erro } // RecordOOM adds info regarding OOM event in the model as an artificial memory sample. -func (cluster *ClusterState) RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { return NewKeyError(containerID.PodID) } @@ -242,11 +237,11 @@ func (cluster *ClusterState) RecordOOM(containerID ContainerID, timestamp time.T return nil } -// AddOrUpdateVpa adds a new VPA with a given ID to the ClusterState if it +// AddOrUpdateVpa adds a new VPA with a given ID to the clusterState if it // didn't yet exist. If the VPA already existed but had a different pod // selector, the pod selector is updated. Updates the links between the VPA and // all aggregations it matches. -func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error { +func (cluster *clusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error { vpaID := VpaID{Namespace: apiObject.Namespace, VpaName: apiObject.Name} annotationsMap := apiObject.Annotations conditionsMap := make(vpaConditionsMap) @@ -258,7 +253,7 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto currentRecommendation = apiObject.Status.Recommendation } - vpa, vpaExists := cluster.Vpas[vpaID] + vpa, vpaExists := cluster.vpas[vpaID] if vpaExists && (vpa.PodSelector.String() != selector.String()) { // Pod selector was changed. Delete the VPA object and recreate // it with the new selector. @@ -269,7 +264,7 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto } if !vpaExists { vpa = NewVpa(vpaID, selector, apiObject.CreationTimestamp.Time) - cluster.Vpas[vpaID] = vpa + cluster.vpas[vpaID] = vpa for aggregationKey, aggregation := range cluster.aggregateStateMap { vpa.UseAggregationIfMatching(aggregationKey, aggregation) } @@ -285,20 +280,36 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto return nil } -// DeleteVpa removes a VPA with the given ID from the ClusterState. -func (cluster *ClusterState) DeleteVpa(vpaID VpaID) error { - vpa, vpaExists := cluster.Vpas[vpaID] +// DeleteVpa removes a VPA with the given ID from the clusterState. +func (cluster *clusterState) DeleteVpa(vpaID VpaID) error { + vpa, vpaExists := cluster.vpas[vpaID] if !vpaExists { return NewKeyError(vpaID) } for _, state := range vpa.aggregateContainerStates { state.MarkNotAutoscaled() } - delete(cluster.Vpas, vpaID) - delete(cluster.EmptyVPAs, vpaID) + delete(cluster.vpas, vpaID) + delete(cluster.emptyVPAs, vpaID) return nil } +func (cluster *clusterState) VPAs() map[VpaID]*Vpa { + return cluster.vpas +} + +func (cluster *clusterState) Pods() map[PodID]*PodState { + return cluster.pods +} + +func (cluster *clusterState) SetObservedVPAs(observedVPAs []*vpa_types.VerticalPodAutoscaler) { + cluster.observedVpas = observedVPAs +} + +func (cluster *clusterState) ObservedVPAs() []*vpa_types.VerticalPodAutoscaler { + return cluster.observedVpas +} + func newPod(id PodID) *PodState { return &PodState{ ID: id, @@ -308,7 +319,7 @@ func newPod(id PodID) *PodState { // getLabelSetKey puts the given labelSet in the global labelSet map and returns a // corresponding labelSetKey. -func (cluster *ClusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { +func (cluster *clusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { labelSetKey := labelSetKey(labelSet.String()) cluster.labelSetMap[labelSetKey] = labelSet return labelSetKey @@ -316,7 +327,7 @@ func (cluster *ClusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { // MakeAggregateStateKey returns the AggregateStateKey that should be used // to aggregate usage samples from a container with the given name in a given pod. -func (cluster *ClusterState) MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey { +func (cluster *clusterState) MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey { return aggregateStateKey{ namespace: pod.ID.Namespace, containerName: containerName, @@ -326,9 +337,9 @@ func (cluster *ClusterState) MakeAggregateStateKey(pod *PodState, containerName } // aggregateStateKeyForContainerID returns the AggregateStateKey for the ContainerID. -// The pod with the corresponding PodID must already be present in the ClusterState. -func (cluster *ClusterState) aggregateStateKeyForContainerID(containerID ContainerID) AggregateStateKey { - pod, podExists := cluster.Pods[containerID.PodID] +// The pod with the corresponding PodID must already be present in the clusterState. +func (cluster *clusterState) aggregateStateKeyForContainerID(containerID ContainerID) AggregateStateKey { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { panic(fmt.Sprintf("Pod not present in the ClusterState: %s/%s", containerID.PodID.Namespace, containerID.PodID.PodName)) } @@ -337,22 +348,22 @@ func (cluster *ClusterState) aggregateStateKeyForContainerID(containerID Contain // findOrCreateAggregateContainerState returns (possibly newly created) AggregateContainerState // that should be used to aggregate usage samples from container with a given ID. -// The pod with the corresponding PodID must already be present in the ClusterState. -func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID ContainerID) *AggregateContainerState { +// The pod with the corresponding PodID must already be present in the clusterState. +func (cluster *clusterState) findOrCreateAggregateContainerState(containerID ContainerID) *AggregateContainerState { aggregateStateKey := cluster.aggregateStateKeyForContainerID(containerID) aggregateContainerState, aggregateStateExists := cluster.aggregateStateMap[aggregateStateKey] if !aggregateStateExists { aggregateContainerState = NewAggregateContainerState() cluster.aggregateStateMap[aggregateStateKey] = aggregateContainerState // Link the new aggregation to the existing VPAs. - for _, vpa := range cluster.Vpas { + for _, vpa := range cluster.vpas { vpa.UseAggregationIfMatching(aggregateStateKey, aggregateContainerState) } } return aggregateContainerState } -// garbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// garbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the clusterState. // AggregateCollectionState is obsolete in following situations: // 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: // @@ -361,7 +372,7 @@ func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID Con // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *clusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { klog.V(1).InfoS("Garbage collection of AggregateCollectionStates triggered") keysToDelete := make([]AggregateStateKey, 0) contributiveKeys := cluster.getContributiveAggregateStateKeys(ctx, controllerFetcher) @@ -379,13 +390,13 @@ func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context } for _, key := range keysToDelete { delete(cluster.aggregateStateMap, key) - for _, vpa := range cluster.Vpas { + for _, vpa := range cluster.vpas { vpa.DeleteAggregation(key) } } } -// RateLimitedGarbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// RateLimitedGarbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the clusterState. // It performs clean up only if more than `gcInterval` passed since the last time it performed a cleanup. // AggregateCollectionState is obsolete in following situations: // 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: @@ -395,7 +406,7 @@ func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *clusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { if now.Sub(cluster.lastAggregateContainerStateGC) < cluster.gcInterval { return } @@ -403,9 +414,9 @@ func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates( cluster.lastAggregateContainerStateGC = now } -func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[AggregateStateKey]bool { +func (cluster *clusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[AggregateStateKey]bool { contributiveKeys := map[AggregateStateKey]bool{} - for _, pod := range cluster.Pods { + for _, pod := range cluster.pods { // Pod is considered contributive in any of following situations: // 1) It is in active state - i.e. not PodSucceeded nor PodFailed. // 2) Its associated controller (e.g. Deployment) still exists. @@ -423,17 +434,17 @@ func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Conte // RecordRecommendation marks the state of recommendation in the cluster. We // keep track of empty recommendations and log information about them // periodically. -func (cluster *ClusterState) RecordRecommendation(vpa *Vpa, now time.Time) error { +func (cluster *clusterState) RecordRecommendation(vpa *Vpa, now time.Time) error { if vpa.Recommendation != nil && len(vpa.Recommendation.ContainerRecommendations) > 0 { - delete(cluster.EmptyVPAs, vpa.ID) + delete(cluster.emptyVPAs, vpa.ID) return nil } - lastLogged, ok := cluster.EmptyVPAs[vpa.ID] + lastLogged, ok := cluster.emptyVPAs[vpa.ID] if !ok { - cluster.EmptyVPAs[vpa.ID] = now + cluster.emptyVPAs[vpa.ID] = now } else { if lastLogged.Add(RecommendationMissingMaxDuration).Before(now) { - cluster.EmptyVPAs[vpa.ID] = now + cluster.emptyVPAs[vpa.ID] = now return fmt.Errorf("VPA %s/%s is missing recommendation for more than %v", vpa.ID.Namespace, vpa.ID.VpaName, RecommendationMissingMaxDuration) } } @@ -442,9 +453,9 @@ func (cluster *ClusterState) RecordRecommendation(vpa *Vpa, now time.Time) error // GetMatchingPods returns a list of currently active pods that match the // given VPA. Traverses through all pods in the cluster - use sparingly. -func (cluster *ClusterState) GetMatchingPods(vpa *Vpa) []PodID { +func (cluster *clusterState) GetMatchingPods(vpa *Vpa) []PodID { matchingPods := []PodID{} - for podID, pod := range cluster.Pods { + for podID, pod := range cluster.pods { if vpa_utils.PodLabelsMatchVPA(podID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { matchingPods = append(matchingPods, podID) @@ -454,7 +465,7 @@ func (cluster *ClusterState) GetMatchingPods(vpa *Vpa) []PodID { } // GetControllerForPodUnderVPA returns controller associated with given Pod. Returns nil if Pod is not controlled by a VPA object. -func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { +func (cluster *clusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { controllingVPA := cluster.GetControllingVPA(pod) if controllingVPA != nil { controller := &controllerfetcher.ControllerKeyWithAPIVersion{ @@ -472,8 +483,8 @@ func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, po } // GetControllingVPA returns a VPA object controlling given Pod. -func (cluster *ClusterState) GetControllingVPA(pod *PodState) *Vpa { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) GetControllingVPA(pod *PodState) *Vpa { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { return vpa @@ -509,3 +520,30 @@ func (k aggregateStateKey) Labels() labels.Labels { } return (*k.labelSetMap)[k.labelSetKey] } + +// ClusterState holds all runtime information about the cluster required for the +// VPA operations, i.e. configuration of resources (pods, containers, +// VPA objects), aggregated utilization of compute resources (CPU, memory) and +// events (container OOMs). +// All input to the VPA Recommender algorithm lives in this structure. +type ClusterState interface { + StateMapSize() int + AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) + GetContainer(containerID ContainerID) *ContainerState + DeletePod(podID PodID) + AddOrUpdateContainer(containerID ContainerID, request Resources) error + AddSample(sample *ContainerUsageSampleWithKey) error + RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error + AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error + DeleteVpa(vpaID VpaID) error + MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey + RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) + RecordRecommendation(vpa *Vpa, now time.Time) error + GetMatchingPods(vpa *Vpa) []PodID + GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion + GetControllingVPA(pod *PodState) *Vpa + VPAs() map[VpaID]*Vpa + SetObservedVPAs([]*vpa_types.VerticalPodAutoscaler) + ObservedVPAs() []*vpa_types.VerticalPodAutoscaler + Pods() map[PodID]*PodState +} diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go index e046b95af3c..c0e67a6524f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go @@ -93,7 +93,7 @@ func TestClusterAddSample(t *testing.T) { assert.NoError(t, cluster.AddSample(makeTestUsageSample())) // Verify that the sample was aggregated into the container stats. - containerStats := cluster.Pods[testPodID].Containers["container-1"] + containerStats := cluster.pods[testPodID].Containers["container-1"] assert.Equal(t, testTimestamp, containerStats.LastCPUSampleStart) } @@ -180,7 +180,7 @@ func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, vpa.aggregateContainerStates) - cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.pods[pod.ID].Phase = apiv1.PodSucceeded cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggregateContainerState should be empty as the pod is no longer active, controller is not alive @@ -211,7 +211,7 @@ func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *te assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, vpa.aggregateContainerStates) - cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.pods[pod.ID].Phase = apiv1.PodSucceeded cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggregateContainerState should not be deleted as the controller is still alive. @@ -342,13 +342,13 @@ func TestMissingKeys(t *testing.T) { assert.EqualError(t, err, "KeyError: {namespace-1 pod-1}") } -func addVpa(cluster *ClusterState, id VpaID, annotations vpaAnnotationsMap, selector string, targetRef *autoscaling.CrossVersionObjectReference) *Vpa { +func addVpa(cluster ClusterState, id VpaID, annotations vpaAnnotationsMap, selector string, targetRef *autoscaling.CrossVersionObjectReference) *Vpa { apiObject := test.VerticalPodAutoscaler().WithNamespace(id.Namespace). WithName(id.VpaName).WithContainer(testContainerID.ContainerName).WithAnnotations(annotations).WithTargetRef(targetRef).Get() return addVpaObject(cluster, id, apiObject, selector) } -func addVpaObject(cluster *ClusterState, id VpaID, vpa *vpa_types.VerticalPodAutoscaler, selector string) *Vpa { +func addVpaObject(cluster ClusterState, id VpaID, vpa *vpa_types.VerticalPodAutoscaler, selector string) *Vpa { labelSelector, _ := metav1.ParseToLabelSelector(selector) parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) err := cluster.AddOrUpdateVpa(vpa, parsedSelector) @@ -356,19 +356,19 @@ func addVpaObject(cluster *ClusterState, id VpaID, vpa *vpa_types.VerticalPodAut klog.ErrorS(err, "AddOrUpdateVpa() failed") os.Exit(255) } - return cluster.Vpas[id] + return cluster.VPAs()[id] } -func addTestVpa(cluster *ClusterState) *Vpa { +func addTestVpa(cluster ClusterState) *Vpa { return addVpa(cluster, testVpaID, testAnnotations, testSelectorStr, testTargetRef) } -func addTestPod(cluster *ClusterState) *PodState { +func addTestPod(cluster ClusterState) *PodState { cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) - return cluster.Pods[testPodID] + return cluster.Pods()[testPodID] } -func addTestContainer(t *testing.T, cluster *ClusterState) *ContainerState { +func addTestContainer(t *testing.T, cluster ClusterState) *ContainerState { err := cluster.AddOrUpdateContainer(testContainerID, testRequest) assert.NoError(t, err) return cluster.GetContainer(testContainerID) @@ -612,7 +612,7 @@ func TestAddOrUpdateVPAPolicies(t *testing.T) { addTestContainer(t, cluster) if tc.oldVpa != nil { oldVpa := addVpaObject(cluster, testVpaID, tc.oldVpa, testSelectorStr) - if !assert.Contains(t, cluster.Vpas, testVpaID) { + if !assert.Contains(t, cluster.vpas, testVpaID) { t.FailNow() } assert.Len(t, oldVpa.aggregateContainerStates, 1, "Expected one container aggregation in VPA %v", testVpaID) @@ -622,7 +622,7 @@ func TestAddOrUpdateVPAPolicies(t *testing.T) { } tc.newVpa.Spec.ResourcePolicy = tc.resourcePolicy addVpaObject(cluster, testVpaID, tc.newVpa, testSelectorStr) - vpa, found := cluster.Vpas[testVpaID] + vpa, found := cluster.vpas[testVpaID] if !assert.True(t, found, "VPA %+v not found in cluster state.", testVpaID) { t.FailNow() } @@ -762,9 +762,9 @@ func TestRecordRecommendation(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cluster := NewClusterState(testGcPeriod) vpa := addVpa(cluster, testVpaID, testAnnotations, testSelectorStr, testTargetRef) - cluster.Vpas[testVpaID].Recommendation = tc.recommendation + cluster.vpas[testVpaID].Recommendation = tc.recommendation if !tc.lastLogged.IsZero() { - cluster.EmptyVPAs[testVpaID] = tc.lastLogged + cluster.emptyVPAs[testVpaID] = tc.lastLogged } err := cluster.RecordRecommendation(vpa, tc.now) @@ -773,10 +773,10 @@ func TestRecordRecommendation(t *testing.T) { } else { assert.NoError(t, err) if tc.expectedEmpty { - assert.Contains(t, cluster.EmptyVPAs, testVpaID) - assert.Equal(t, cluster.EmptyVPAs[testVpaID], tc.expectedLastLogged) + assert.Contains(t, cluster.emptyVPAs, testVpaID) + assert.Equal(t, cluster.emptyVPAs[testVpaID], tc.expectedLastLogged) } else { - assert.NotContains(t, cluster.EmptyVPAs, testVpaID) + assert.NotContains(t, cluster.emptyVPAs, testVpaID) } } }) @@ -924,7 +924,7 @@ func TestVPAWithMatchingPods(t *testing.T) { containerID := ContainerID{testPodID, "foo"} assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) } - assert.Equal(t, tc.expectedMatch, cluster.Vpas[vpa.ID].PodCount) + assert.Equal(t, tc.expectedMatch, cluster.vpas[vpa.ID].PodCount) }) } // Run with adding Pods first @@ -937,7 +937,7 @@ func TestVPAWithMatchingPods(t *testing.T) { assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) } vpa := addVpa(cluster, testVpaID, testAnnotations, tc.vpaSelector, testTargetRef) - assert.Equal(t, tc.expectedMatch, cluster.Vpas[vpa.ID].PodCount) + assert.Equal(t, tc.expectedMatch, cluster.vpas[vpa.ID].PodCount) }) } } diff --git a/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go b/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go index 1f456a80c18..273a2093f3f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go +++ b/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go @@ -43,7 +43,7 @@ type Recommender interface { // RunOnce performs one iteration of recommender duties followed by update of recommendations in VPA objects. RunOnce() // GetClusterState returns ClusterState used by Recommender - GetClusterState() *model.ClusterState + GetClusterState() model.ClusterState // GetClusterStateFeeder returns ClusterStateFeeder used by Recommender GetClusterStateFeeder() input.ClusterStateFeeder // UpdateVPAs computes recommendations and sends VPAs status updates to API Server @@ -55,7 +55,7 @@ type Recommender interface { } type recommender struct { - clusterState *model.ClusterState + clusterState model.ClusterState clusterStateFeeder input.ClusterStateFeeder checkpointWriter checkpoint.CheckpointWriter checkpointsGCInterval time.Duration @@ -68,7 +68,7 @@ type recommender struct { recommendationPostProcessor []RecommendationPostProcessor } -func (r *recommender) GetClusterState() *model.ClusterState { +func (r *recommender) GetClusterState() model.ClusterState { return r.clusterState } @@ -81,13 +81,13 @@ func (r *recommender) UpdateVPAs() { cnt := metrics_recommender.NewObjectCounter() defer cnt.Observe() - for _, observedVpa := range r.clusterState.ObservedVpas { + for _, observedVpa := range r.clusterState.ObservedVPAs() { key := model.VpaID{ Namespace: observedVpa.Namespace, VpaName: observedVpa.Name, } - vpa, found := r.clusterState.Vpas[key] + vpa, found := r.clusterState.VPAs()[key] if !found { continue } @@ -155,7 +155,7 @@ func (r *recommender) RunOnce() { r.clusterStateFeeder.LoadRealTimeMetrics() timer.ObserveStep("LoadMetrics") - klog.V(3).InfoS("ClusterState is tracking", "pods", len(r.clusterState.Pods), "vpas", len(r.clusterState.Vpas)) + klog.V(3).InfoS("ClusterState is tracking", "pods", len(r.clusterState.Pods()), "vpas", len(r.clusterState.VPAs())) r.UpdateVPAs() timer.ObserveStep("UpdateVPAs") @@ -172,7 +172,7 @@ func (r *recommender) RunOnce() { // RecommenderFactory makes instances of Recommender. type RecommenderFactory struct { - ClusterState *model.ClusterState + ClusterState model.ClusterState ClusterStateFeeder input.ClusterStateFeeder ControllerFetcher controllerfetcher.ControllerFetcher From 481f8db1167923fdb8880d4884a66c14010caf71 Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Wed, 5 Mar 2025 18:00:13 +0100 Subject: [PATCH 08/39] Store InitContainers in PodState --- .../pkg/recommender/input/cluster_feeder.go | 5 ++ .../recommender/input/cluster_feeder_test.go | 65 ++++++++++++++++++- .../pkg/recommender/input/spec/spec_client.go | 18 +++-- .../input/spec/spec_client_test_util.go | 22 +++++-- .../pkg/recommender/model/cluster.go | 2 + 5 files changed, 98 insertions(+), 14 deletions(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index 1cd82a9a765..49d5fa88dca 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -476,6 +476,11 @@ func (feeder *clusterStateFeeder) LoadPods() { klog.V(0).InfoS("Failed to add container", "container", container.ID, "error", err) } } + for _, initContainer := range pod.InitContainers { + podInitContainers := feeder.clusterState.Pods()[pod.ID].InitContainers + feeder.clusterState.Pods()[pod.ID].InitContainers = append(podInitContainers, initContainer.ID.ContainerName) + + } } } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index dcc311671f8..e6ad3dec616 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -419,7 +419,70 @@ func makeTestSpecClient(podLabels []map[string]string) spec.SpecClient { } } -func TestClusterStateFeeder_LoadPods(t *testing.T) { +func newTestContainerSpec(podID model.PodID, containerName string, milicores int, memory int64) spec.BasicContainerSpec { + containerID := model.ContainerID{ + PodID: podID, + ContainerName: containerName, + } + requestedResources := model.Resources{ + model.ResourceCPU: model.ResourceAmount(milicores), + model.ResourceMemory: model.ResourceAmount(memory), + } + return spec.BasicContainerSpec{ + ID: containerID, + Image: containerName + "Image", + Request: requestedResources, + } +} + +func newTestPodSpec(podId model.PodID, containerSpecs []spec.BasicContainerSpec, initContainerSpecs []spec.BasicContainerSpec) *spec.BasicPodSpec { + return &spec.BasicPodSpec{ + ID: podId, + PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, + Containers: containerSpecs, + InitContainers: initContainerSpecs, + } +} + +func TestClusterStateFeeder_LoadPods_ContainerTracking(t *testing.T) { + podWithoutInitContainersID := model.PodID{Namespace: "default", PodName: "PodWithoutInitContainers"} + containerSpecs := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithoutInitContainersID, "container1", 500, 512*1024*1024), + newTestContainerSpec(podWithoutInitContainersID, "container2", 1000, 1024*1024*1024), + } + podWithoutInitContainers := newTestPodSpec(podWithoutInitContainersID, containerSpecs, nil) + + podWithInitContainersID := model.PodID{Namespace: "default", PodName: "PodWithInitContainers"} + containerSpecs2 := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithInitContainersID, "container1", 2000, 2048*1024*1024), + } + initContainerSpecs2 := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithInitContainersID, "init1", 40, 128*1024*1024), + newTestContainerSpec(podWithInitContainersID, "init2", 100, 256*1024*1024), + } + podWithInitContainers := newTestPodSpec(podWithInitContainersID, containerSpecs2, initContainerSpecs2) + + client := &testSpecClient{pods: []*spec.BasicPodSpec{podWithoutInitContainers, podWithInitContainers}} + + clusterState := model.NewClusterState(testGcPeriod) + + feeder := clusterStateFeeder{ + specClient: client, + memorySaveMode: false, + clusterState: clusterState, + } + + feeder.LoadPods() + + assert.Equal(t, len(feeder.clusterState.Pods()), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithInitContainersID].Containers), 1) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithInitContainersID].InitContainers), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithoutInitContainersID].Containers), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithoutInitContainersID].InitContainers), 0) + +} + +func TestClusterStateFeeder_LoadPods_MemorySaverMode(t *testing.T) { for _, tc := range []struct { Name string VPALabelSelectors []string diff --git a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go index dba046b871a..81de7307358 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go @@ -32,6 +32,8 @@ type BasicPodSpec struct { PodLabels map[string]string // List of containers within this pod. Containers []BasicContainerSpec + // List of init containers within this pod. + InitContainers []BasicContainerSpec // PodPhase describing current life cycle phase of the Pod. Phase v1.PodPhase } @@ -82,21 +84,23 @@ func newBasicPodSpec(pod *v1.Pod) *BasicPodSpec { PodName: pod.Name, Namespace: pod.Namespace, } - containerSpecs := newContainerSpecs(podId, pod) + containerSpecs := newContainerSpecs(podId, pod.Spec.Containers) + initContainerSpecs := newContainerSpecs(podId, pod.Spec.InitContainers) basicPodSpec := &BasicPodSpec{ - ID: podId, - PodLabels: pod.Labels, - Containers: containerSpecs, - Phase: pod.Status.Phase, + ID: podId, + PodLabels: pod.Labels, + Containers: containerSpecs, + InitContainers: initContainerSpecs, + Phase: pod.Status.Phase, } return basicPodSpec } -func newContainerSpecs(podID model.PodID, pod *v1.Pod) []BasicContainerSpec { +func newContainerSpecs(podID model.PodID, containers []v1.Container) []BasicContainerSpec { var containerSpecs []BasicContainerSpec - for _, container := range pod.Spec.Containers { + for _, container := range containers { containerSpec := newContainerSpec(podID, container) containerSpecs = append(containerSpecs, containerSpec) } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go index 4a2a212e498..e6cfc606ab1 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go @@ -82,6 +82,13 @@ spec: requests: memory: "4096Mi" cpu: "4000m" + initContainers: + - name: Name21-init + image: Name21-initImage + resources: + requests: + memory: "128Mi" + cpu: "40m" ` type podListerMock struct { @@ -116,8 +123,10 @@ func newSpecClientTestCase() *specClientTestCase { containerSpec21 := newTestContainerSpec(podID2, "Name21", 2000, 2048*1024*1024) containerSpec22 := newTestContainerSpec(podID2, "Name22", 4000, 4096*1024*1024) - podSpec1 := newTestPodSpec(podID1, containerSpec11, containerSpec12) - podSpec2 := newTestPodSpec(podID2, containerSpec21, containerSpec22) + initContainerSpec21 := newTestContainerSpec(podID2, "Name21-init", 40, 128*1024*1024) + + podSpec1 := newTestPodSpec(podID1, []BasicContainerSpec{containerSpec11, containerSpec12}, nil) + podSpec2 := newTestPodSpec(podID2, []BasicContainerSpec{containerSpec21, containerSpec22}, []BasicContainerSpec{initContainerSpec21}) return &specClientTestCase{ podSpecs: []*BasicPodSpec{podSpec1, podSpec2}, @@ -141,11 +150,12 @@ func newTestContainerSpec(podID model.PodID, containerName string, milicores int } } -func newTestPodSpec(podId model.PodID, containerSpecs ...BasicContainerSpec) *BasicPodSpec { +func newTestPodSpec(podId model.PodID, containerSpecs []BasicContainerSpec, initContainerSpecs []BasicContainerSpec) *BasicPodSpec { return &BasicPodSpec{ - ID: podId, - PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, - Containers: containerSpecs, + ID: podId, + PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, + Containers: containerSpecs, + InitContainers: initContainerSpecs, } } diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go index c6c6ca674af..fcd79d6ad7e 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go @@ -88,6 +88,8 @@ type PodState struct { labelSetKey labelSetKey // Containers that belong to the Pod, keyed by the container name. Containers map[string]*ContainerState + // InitContainers is a list of init containers names which belong to the Pod. + InitContainers []string // PodPhase describing current life cycle phase of the Pod. Phase apiv1.PodPhase } From 3c7689f68c1eccda91bb6ab990a8e947019a856f Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Wed, 5 Mar 2025 18:02:30 +0100 Subject: [PATCH 09/39] Drop MetricSamples for InitContainers --- .../pkg/recommender/input/cluster_feeder.go | 6 ++ .../recommender/input/cluster_feeder_test.go | 92 ++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index 49d5fa88dca..8d7855b385f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -493,6 +493,12 @@ func (feeder *clusterStateFeeder) LoadRealTimeMetrics() { sampleCount := 0 droppedSampleCount := 0 for _, containerMetrics := range containersMetrics { + podInitContainers := feeder.clusterState.Pods()[containerMetrics.ID.PodID].InitContainers + if slices.Contains(podInitContainers, containerMetrics.ID.ContainerName) { + klog.V(3).InfoS("Skipping metric samples for init container", "pod", klog.KRef(containerMetrics.ID.PodID.Namespace, containerMetrics.ID.PodID.PodName), "container", containerMetrics.ID.ContainerName) + droppedSampleCount += len(containerMetrics.Usage) + continue + } for _, sample := range newContainerUsageSamplesWithKey(containerMetrics) { if err := feeder.clusterState.AddSample(sample); err != nil { // Not all pod states are tracked in memory saver mode diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index e6ad3dec616..aaffed0548d 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -36,6 +36,7 @@ import ( vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" fakeautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/fake" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/metrics" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/spec" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" @@ -556,11 +557,100 @@ func TestClusterStateFeeder_LoadPods_MemorySaverMode(t *testing.T) { } feeder.LoadPods() - assert.Len(t, feeder.clusterState.Pods, len(tc.PodLabels), "number of pods is not %d", len(tc.PodLabels)) + assert.Len(t, clusterState.addedPods, len(tc.PodLabels), "number of pods is not %d", len(tc.PodLabels)) }) } } +func newContainerMetricsSnapshot(id model.ContainerID, cpuUsage int64, memUsage int64) (*metrics.ContainerMetricsSnapshot, []*model.ContainerUsageSampleWithKey) { + snapshotTimestamp := time.Now() + snapshotWindow := time.Duration(1234) + snapshot := &metrics.ContainerMetricsSnapshot{ + ID: id, + SnapshotTime: snapshotTimestamp, + SnapshotWindow: snapshotWindow, + Usage: model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuUsage), + model.ResourceMemory: model.ResourceAmount(memUsage), + }, + } + samples := []*model.ContainerUsageSampleWithKey{ + { + Container: id, + ContainerUsageSample: model.ContainerUsageSample{ + MeasureStart: snapshotTimestamp, + Resource: model.ResourceCPU, + Usage: model.ResourceAmount(cpuUsage), + }, + }, + { + Container: id, + ContainerUsageSample: model.ContainerUsageSample{ + MeasureStart: snapshotTimestamp, + Resource: model.ResourceMemory, + Usage: model.ResourceAmount(memUsage), + }, + }, + } + return snapshot, samples +} + +type fakeMetricsClient struct { + snapshots []*metrics.ContainerMetricsSnapshot +} + +func (m fakeMetricsClient) GetContainersMetrics() ([]*metrics.ContainerMetricsSnapshot, error) { + return m.snapshots, nil +} + +func TestClusterStateFeeder_LoadRealTimeMetrics(t *testing.T) { + namespaceName := "test-namespace" + podID := model.PodID{Namespace: namespaceName, PodName: "Pod"} + regularContainer1 := model.ContainerID{PodID: podID, ContainerName: "Container1"} + regularContainer2 := model.ContainerID{PodID: podID, ContainerName: "Container2"} + initContainer := model.ContainerID{PodID: podID, ContainerName: "InitContainer"} + + pods := map[model.PodID]*model.PodState{ + podID: {ID: podID, + Containers: map[string]*model.ContainerState{ + "Container1": {}, + "Container2": {}, + }, + InitContainers: []string{ + "InitContainer", + }}, + } + + var containerMetricsSnapshots []*metrics.ContainerMetricsSnapshot + + regularContainer1MetricsSnapshot, regularContainer1UsageSamples := newContainerMetricsSnapshot(regularContainer1, 100, 1024) + containerMetricsSnapshots = append(containerMetricsSnapshots, regularContainer1MetricsSnapshot) + regularContainer2MetricsSnapshot, regularContainer2UsageSamples := newContainerMetricsSnapshot(regularContainer2, 200, 2048) + containerMetricsSnapshots = append(containerMetricsSnapshots, regularContainer2MetricsSnapshot) + initContainer1MetricsSnapshots, _ := newContainerMetricsSnapshot(initContainer, 300, 3072) + containerMetricsSnapshots = append(containerMetricsSnapshots, initContainer1MetricsSnapshots) + + clusterState := NewFakeClusterState(nil, pods) + + feeder := clusterStateFeeder{ + memorySaveMode: false, + clusterState: clusterState, + metricsClient: fakeMetricsClient{snapshots: containerMetricsSnapshots}, + } + + feeder.LoadRealTimeMetrics() + + assert.Equal(t, 2, len(clusterState.addedSamples)) + + samplesForContainer1 := clusterState.addedSamples[regularContainer1] + assert.Contains(t, samplesForContainer1, regularContainer1UsageSamples[0]) + assert.Contains(t, samplesForContainer1, regularContainer1UsageSamples[1]) + + samplesForContainer2 := clusterState.addedSamples[regularContainer2] + assert.Contains(t, samplesForContainer2, regularContainer2UsageSamples[0]) + assert.Contains(t, samplesForContainer2, regularContainer2UsageSamples[1]) +} + type fakeHistoryProvider struct { history map[model.PodID]*history.PodHistory err error From 33871fa816f5f8186e7b701010ff70e087b9ee51 Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Wed, 12 Mar 2025 17:39:10 +0100 Subject: [PATCH 10/39] Address review feedback --- .../pkg/recommender/model/cluster.go | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go index fcd79d6ad7e..be8eceaed86 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go @@ -35,6 +35,33 @@ const ( RecommendationMissingMaxDuration = 30 * time.Minute ) +// ClusterState holds all runtime information about the cluster required for the +// VPA operations, i.e. configuration of resources (pods, containers, +// VPA objects), aggregated utilization of compute resources (CPU, memory) and +// events (container OOMs). +// All input to the VPA Recommender algorithm lives in this structure. +type ClusterState interface { + StateMapSize() int + AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) + GetContainer(containerID ContainerID) *ContainerState + DeletePod(podID PodID) + AddOrUpdateContainer(containerID ContainerID, request Resources) error + AddSample(sample *ContainerUsageSampleWithKey) error + RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error + AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error + DeleteVpa(vpaID VpaID) error + MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey + RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) + RecordRecommendation(vpa *Vpa, now time.Time) error + GetMatchingPods(vpa *Vpa) []PodID + GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion + GetControllingVPA(pod *PodState) *Vpa + VPAs() map[VpaID]*Vpa + SetObservedVPAs([]*vpa_types.VerticalPodAutoscaler) + ObservedVPAs() []*vpa_types.VerticalPodAutoscaler + Pods() map[PodID]*PodState +} + type clusterState struct { // Pods in the cluster. pods map[PodID]*PodState @@ -45,7 +72,7 @@ type clusterState struct { // a warning about it. emptyVPAs map[VpaID]time.Time // Observed VPAs. Used to check if there are updates needed. - observedVpas []*vpa_types.VerticalPodAutoscaler + observedVPAs []*vpa_types.VerticalPodAutoscaler // All container aggregations where the usage samples are stored. aggregateStateMap aggregateContainerStatesMap @@ -305,11 +332,11 @@ func (cluster *clusterState) Pods() map[PodID]*PodState { } func (cluster *clusterState) SetObservedVPAs(observedVPAs []*vpa_types.VerticalPodAutoscaler) { - cluster.observedVpas = observedVPAs + cluster.observedVPAs = observedVPAs } func (cluster *clusterState) ObservedVPAs() []*vpa_types.VerticalPodAutoscaler { - return cluster.observedVpas + return cluster.observedVPAs } func newPod(id PodID) *PodState { @@ -522,30 +549,3 @@ func (k aggregateStateKey) Labels() labels.Labels { } return (*k.labelSetMap)[k.labelSetKey] } - -// ClusterState holds all runtime information about the cluster required for the -// VPA operations, i.e. configuration of resources (pods, containers, -// VPA objects), aggregated utilization of compute resources (CPU, memory) and -// events (container OOMs). -// All input to the VPA Recommender algorithm lives in this structure. -type ClusterState interface { - StateMapSize() int - AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) - GetContainer(containerID ContainerID) *ContainerState - DeletePod(podID PodID) - AddOrUpdateContainer(containerID ContainerID, request Resources) error - AddSample(sample *ContainerUsageSampleWithKey) error - RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error - AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error - DeleteVpa(vpaID VpaID) error - MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey - RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) - RecordRecommendation(vpa *Vpa, now time.Time) error - GetMatchingPods(vpa *Vpa) []PodID - GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion - GetControllingVPA(pod *PodState) *Vpa - VPAs() map[VpaID]*Vpa - SetObservedVPAs([]*vpa_types.VerticalPodAutoscaler) - ObservedVPAs() []*vpa_types.VerticalPodAutoscaler - Pods() map[PodID]*PodState -} From bd363cdeace9ebc3867c7508b2d29019f3c6adc4 Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Wed, 12 Mar 2025 15:51:16 -0400 Subject: [PATCH 11/39] Address comments and wrap lines --- .../7862-cpu-startup-boost/README.md | 209 +++++++++++++----- 1 file changed, 153 insertions(+), 56 deletions(-) diff --git a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md index cd2267f4594..261d62b989e 100644 --- a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md +++ b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md @@ -10,11 +10,15 @@ - [Workflow](#workflow) - [API Changes](#api-changes) - [Priority of `StartupBoost`](#priority-of-startupboost) - - [CPU startup boost and VPA autoscaling modes](#cpu-startup-boost-and-vpa-autoscaling-modes) - [Validation](#validation) - - [Startup Boost and Readiness/Startup Probes](#startup-boost-and-readinessstartup-probes) - - [Failed in-place downsizes](#failed-in-place-downsizes) + - [Static Validation](#static-validation) + - [Dynamic Validation](#dynamic-validation) + - [Feature Enablement](#feature-enablement) + - [Mitigating Failed In-Place Downsizes](#mitigating-failed-in-place-downsizes) - [Test Plan](#test-plan) + - [Examples](#examples) + - [CPU Boost Only](#cpu-boost-only) + - [CPU Boost and Vanilla VPA](#cpu-boost-and-vanilla-vpa) - [Implementation History](#implementation-history) @@ -34,58 +38,153 @@ leveraging the [in-place pod resize Kubernetes feature](https://github.com/kuber ### Goals -* Allow VPA to boost the CPU request and limit of a pod's containers when the pod during the pod startup (from creation time until it becomes `Ready`.) -* Allow VPA to scale pods down to the original CPU resource values, [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources), as soon as their [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) condition is true. +* Allow VPA to boost the CPU request and limit of a pod's containers during the +pod startup (from creation time until it becomes `Ready`). +* Allow VPA to scale pods down [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources) +to the existing VPA recommendation for that container, if any, or to the CPU +resources configured in the pod spec, as soon as their [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) +condition is true. ### Non-Goals * Allow VPA to boost CPU resources of pods that are already [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions). -* Allow VPA to boost CPU resources during startup of workloads that have not configured a [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). +* Allow VPA to boost CPU resources during startup of workloads that have not +configured a [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). * Allow VPA to boost memory resources. + * This is out of scope for now because the in-place pod resize feature + [does not support memory limit decrease yet.](https://github.com/kubernetes/enhancements/tree/758ea034908515a934af09d03a927b24186af04c/keps/sig-node/1287-in-place-update-pod-resources#memory-limit-decreases) ## Proposal -To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190) with a `StartupBoost.CPU.Factor` field, that will instruct VPA the factor by which to multiply the initial resource request and limit of the containers' targeted by the VPA object. +* To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191) +with a new `StartupBoost` field to allow users to configure the CPU startup +boost. -This AEP also proposes to extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L230-L235) with a new `StartupBoostOnly` mode to allow users to only enable the startup boost feature and not vanilla VPA altogether. +* To extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) +with a new `StartupBoostOnly` mode to allow users to only enable the startup +boost feature and not vanilla VPA altogether. ## Design Details ### Workflow -1. The user first configures the CPU startup boost factor on their VPA object +1. The user first configures the CPU startup boost on their VPA object 1. When a pod targeted by that VPA is created, the kube-apiserver invokes the VPA Admission Controller 1. The VPA Admission Controller modifies the pod's containers CPU request and -limits to align with its `boostPolicy`, if specified, during the -pod creation. - * VPA Admission Controller will not modify the CPU resources of a pod if - it does not configure a Readiness or a Startup probe. +limits to align with its `StartupBoost` policy, if specified, during the pod +creation. 1. The VPA Updater monitors pods targeted by the VPA object and when the pod condition is `Ready`, it scales down the CPU resources to the appropriate -non-boosted value: `max(existing VPA recommendation for that container, minimum configured value)`. +non-boosted value: `existing VPA recommendation for that container` (if any) OR +the `CPU resources configured in the pod spec`. * The scale down is applied [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). ### API Changes -To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190) with a `StartupBoost.CPU.Factor` field, that will instruct VPA the factor by which to multiply the initial resource request and limit of the containers' targeted by the VPA object. +The new `StartupBoost` parameter will be added to the [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191) +and contain the following fields: + * `StartupBoost.CPU.Factor`: the factor by which to multiply the initial + resource request and limit of the containers' targeted by the VPA object. + * `StartupBoost.CPU.Value`: the target value of the CPU request or limit + during the startup boost phase. -This AEP also proposes to extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L230-L235) with a new `StartupBoostOnly` mode to allow users to only enable the startup boost feature and not vanilla VPA altogether. +NOTE: The boosted CPU value will be capped by +[`--container-recommendation-max-allowed-cpu`](https://github.com/kubernetes/autoscaler/blob/4d294562e505431d518a81e8833accc0ec99c9b8/vertical-pod-autoscaler/pkg/recommender/main.go#L122) +flag value, if set. +NOTE: Only one of `Factor` or `Value` may be specified per container policy. #### Priority of `StartupBoost` -The new `StartupBoost` field will take precedence over the rest of the container resource policy configurations. Functioning independently from all other fields in [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L190), except for [`ContainerName`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191-L194) and [`Mode`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L195-L197). This means that a container's CPU request/limit can be boosted during startup beyond [`MaxAllowed`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L202-L205), for example, or it will be able to be boosted even if -CPU is explicitly excluded from [`ControlledResources`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L207-L211). +The new `StartupBoost` field will take precedence over the rest of the container +resource policy configurations. Functioning independently from all other fields +in [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191), +**except for**: + * [`ContainerName`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L192-L195) + * [`Mode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L196-L198) + * [`ControlledValues`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L214-L217) -#### CPU startup boost and VPA autoscaling modes +This means that a container's CPU request/limit can be boosted during startup +beyond [`MaxAllowed`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L203-L206), +for example, or it will be able to be boosted even if CPU is explicitly +excluded from [`ControlledResources`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L208-L212). -Initially, CPU startup boost will only be able to be used if the container or VPA scaling mode is [`InPlaceOrRecreate`](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L61), since it does not make sense to use this feature with disruptive modes of VPA +### Validation + +#### Static Validation + +* We will check that the `startupBoost` configuration is valid when VPA objects +are created/updated: + * The VPA autoscaling mode must be `InPlaceOrRecreate` (since it does not + make sense to use this feature with disruptive modes of VPA). + * The boost factor is >= 1 + * Only one of `StartupBoost.CPU.Factor` or `StartupBoost.CPU.Value` is + specified + +#### Dynamic Validation + +* `StartupBoost.CPU.Value` must be greater than the CPU request or limit of the + container during the boost phase, otherwise we risk downscaling the container. + +* Workloads must be configured with a Readiness or a Startup probe to be able to +utilize this feature. Therefore, VPA will not boost CPU resources of workloads +that do not configure a Readiness or a Startup probe. + +### Feature Enablement -Here's an example of the VPA CRD, incorporating CPU boosting (only shows relevant sections): +During the Alpha launch of this feature, users will need to: +* Enable the CPU startup boost feature via a binary flag. +* Set the VPA autoscaling mode to `InPlaceOrRecreate` +* Set the [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) +to either: + * **NEW**: `StartupBoostOnly`: this mode will allow users to only enable the + startup boost feature for a container and not vanilla VPA altogether. + * **NEW**: `Auto`: in this mode, both vanilla VPA and CPU Startup Boost will + be enabled. + + +### Mitigating Failed In-Place Downsizes + +The VPA Updater will evict a pod to actuate a startup CPU boost +recommendation if it attempted to apply the recommendation in place and it +failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169) +where the VPA updater will consider that the update failed). + +We will cache pods that have failed the in-place downsize and not reattempt to +CPU boost them again for the next hour, to **avoid** an eviction loop scenario +like this: + +1. A pod is created and has its CPU resources boosted +2. The pod is ready. VPA Updater tries to downscale the pod in-place and it +fails. +1. VPA Updater evicts the pod. (If we do nothing, the logic flow goes back to +(1)). + +## Test Plan + +Other than comprehensive unit tests, we will also add the following scenarios to +our e2e tests: + +* CPU Startup Boost recommendation is applied to pod controlled by VPA until it +becomes `Ready`. Then, the pod is scaled back down in-place. + * Boost is applied to all containers of a pod. + * Boost is applied to a subset of containers. +* CPU Startup Boost will not be applied if a pod is not configured with a +Readiness or a Startup probe. +* Pod is evicted the first time that an in-place update fails when scaling the +pod back down. And a new CPU boost is not attempted when the pod is recreated. + + +## Examples + +Here are some examples of the VPA CR incorporating CPU boosting for different +scenarios. + +### CPU Boost Only ```yaml apiVersion: "autoscaling.k8s.io/v1" @@ -98,51 +197,49 @@ spec: kind: Deployment name: example updatePolicy: + # VPA Update mode must be InPlaceOrRecreate updateMode: "InPlaceOrRecreate" resourcePolicy: containerPolicies: - - containerName: 'boosted-container' - mode: 'StartupBoostOnly' + - containerName: "boosted-container-name" + mode: "StartupBoostOnly" startupBoost: cpu: factor: 2.0 ``` -### Validation - -* We'll also validate that the `startupBoost` configuration is valid when VPA objects are created/updated: - * The VPA autoscaling mode must be `InPlaceOrRecreate` - * The boost factor is >= 1 - -### Startup Boost and Readiness/Startup Probes - -The CPU Startup Boost functionality will boost the CPU requests of containers -until the pod has a `Ready` condition. Therefore, in this version of the AEP, -workloads must be configured with a Readiness or a Startup probe to be able to -utilize this feature. - -### Failed in-place downsizes - -The VPA Updater **will not** evict a pod to actuate a startup CPU boost recommendation if it attempted to apply the recommendation in place and it failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA updater will consider that the update failed). This is to avoid an eviction loop: - -1. A pod is created and has its CPU resources boosted -1. The pod is ready. VPA Updater tries to downscale the pod in-place and it fails. -1. VPA Updater evicts the pod. Logic flow goes back to (1). - -## Test Plan - -* Add tests to verify the VPA fields validation - -We will add the following scenarios to our e2e tests: - -* CPU Startup Boost recommendation is applied to pod controlled by VPA until it becomes `Ready`. Then, the pod is scaled back down in-place. - * Boost is applied to all containers of a pod. - * Boost is applied to a subset of containers. -* CPU Startup Boost will not be applied if a pod is not configured with a Readiness or a Startup probe. -* Pod should not be evicted if the in-place update fails when scaling the pod back down. +### CPU Boost and Vanilla VPA +```yaml +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: example-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: example + updatePolicy: + # VPA Update mode must be InPlaceOrRecreate + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + - containerName: "boosted-container-name" + mode: "Auto" # Vanilla VPA mode + Startup Boost + minAllowed: + cpu: "250m" + memory: "100Mi" + maxAllowed: + cpu: "500m" + memory: "600Mi" + # The CPU boosted resources can go beyond maxAllowed. + startupBoost: + cpu: + value: 4 +``` ## Implementation History -* 2025-02-24: Initial version. +* 2025-03-12: Initial version. From 0f5fe4254e1c91c236ac5ea039b528282aa0c134 Mon Sep 17 00:00:00 2001 From: Adrian Moisey Date: Mon, 17 Mar 2025 11:30:29 +0200 Subject: [PATCH 12/39] Add adrianmoisey to VPA approvers --- vertical-pod-autoscaler/OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/vertical-pod-autoscaler/OWNERS b/vertical-pod-autoscaler/OWNERS index 72107eb3bd5..bc51f4f1e34 100644 --- a/vertical-pod-autoscaler/OWNERS +++ b/vertical-pod-autoscaler/OWNERS @@ -3,6 +3,7 @@ approvers: - jbartosik - voelzmo - raywainman +- adrianmoisey reviewers: - kwiesmueller - jbartosik From 29d9088a9948490ba2d14c98f4ce11d0e7003bb7 Mon Sep 17 00:00:00 2001 From: Ray Wainman Date: Mon, 17 Mar 2025 15:24:08 +0000 Subject: [PATCH 13/39] change our release process to bump the version in the main branch immediately after cutting a release branch so that new development is done against the new version --- vertical-pod-autoscaler/RELEASE.md | 15 +++++++-------- vertical-pod-autoscaler/common/version.go | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/vertical-pod-autoscaler/RELEASE.md b/vertical-pod-autoscaler/RELEASE.md index 9b7fc80b981..19d5b1c0ae0 100644 --- a/vertical-pod-autoscaler/RELEASE.md +++ b/vertical-pod-autoscaler/RELEASE.md @@ -9,7 +9,7 @@ Before doing the release for the first time check if you have all the necessary There are the following steps of the release process: 1. [ ] Open issue to track the release. -2. [ ] Update VPA version const. +2. [ ] Rollup all changes. 3. [ ] Build and stage images. 4. [ ] Test the release. 5. [ ] Promote image. @@ -20,7 +20,7 @@ There are the following steps of the release process: Open a new issue to track the release, use the [vpa_release](https://github.com/kubernetes/autoscaler/issues/new?&template=vpa_release.md) template. We use the issue to communicate what is state of the release. -## Update VPA version const +## Rollup all changes 1. [ ] Wait for all VPA changes that will be in the release to merge. 2. [ ] Wait for [the end to end tests](https://testgrid.k8s.io/sig-autoscaling-vpa) to run with all VPA changes @@ -31,13 +31,12 @@ We use the issue to communicate what is state of the release. ### New minor release -1. [ ] Change the version in - [common/version-go](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/common/version.go) - to `1.${next-minor}.0`, -2. [ ] Commit and merge the change, -3. [ ] Go to the merged change, -4. [ ] [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) named `vpa-release-1.${next-minor}` from the +1. [ ] [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) named `vpa-release-1.${next-minor}` from the merged change. +2. [ ] In the **main branch**, change the version in + [common/version-go](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/common/version.go) + to `1.${next-minor}.0`. +3. [ ] Commit and merge the change. ### New patch release diff --git a/vertical-pod-autoscaler/common/version.go b/vertical-pod-autoscaler/common/version.go index cfd4803cd8e..06a88725a66 100644 --- a/vertical-pod-autoscaler/common/version.go +++ b/vertical-pod-autoscaler/common/version.go @@ -21,7 +21,7 @@ package common var gitCommit = "" // versionCore is the version of VPA. -const versionCore = "1.3.0" +const versionCore = "1.4.0" // VerticalPodAutoscalerVersion returns the version of the VPA. func VerticalPodAutoscalerVersion() string { From 003e6cd67cd01b0d5aadebfb935628618fbc0ab6 Mon Sep 17 00:00:00 2001 From: elmiko Date: Wed, 5 Mar 2025 12:43:04 -0500 Subject: [PATCH 14/39] make DecreaseTargetSize more accurate for clusterapi this change ensures that when DecreaseTargetSize is counting the nodes that it does not include any instances which are considered to be pending (i.e. not having a node ref), deleting, or are failed. this change will allow the core autoscaler to then decrease the size of the node group accordingly, instead of raising an error. This change also add some code to the unit tests to make detection of this condition easier. --- .../clusterapi/clusterapi_controller.go | 23 +++ .../clusterapi/clusterapi_controller_test.go | 113 ++++++++++++ .../clusterapi/clusterapi_nodegroup.go | 23 ++- .../clusterapi/clusterapi_nodegroup_test.go | 161 ++++++++++++++---- 4 files changed, 283 insertions(+), 37 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index 967177cafa6..ab2095ef594 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -55,6 +55,7 @@ const ( resourceNameMachineSet = "machinesets" resourceNameMachineDeployment = "machinedeployments" resourceNameMachinePool = "machinepools" + deletingMachinePrefix = "deleting-machine-" failedMachinePrefix = "failed-machine-" pendingMachinePrefix = "pending-machine-" machineTemplateKind = "MachineTemplate" @@ -314,6 +315,9 @@ func (c *machineController) findMachineByProviderID(providerID normalizedProvide return u.DeepCopy(), nil } + if isDeletingMachineProviderID(providerID) { + return c.findMachine(machineKeyFromDeletingMachineProviderID(providerID)) + } if isFailedMachineProviderID(providerID) { return c.findMachine(machineKeyFromFailedProviderID(providerID)) } @@ -339,6 +343,19 @@ func (c *machineController) findMachineByProviderID(providerID normalizedProvide return c.findMachine(path.Join(ns, machineID)) } +func createDeletingMachineNormalizedProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", deletingMachinePrefix, namespace, name) +} + +func isDeletingMachineProviderID(providerID normalizedProviderID) bool { + return strings.HasPrefix(string(providerID), deletingMachinePrefix) +} + +func machineKeyFromDeletingMachineProviderID(providerID normalizedProviderID) string { + namespaceName := strings.TrimPrefix(string(providerID), deletingMachinePrefix) + return strings.Replace(namespaceName, "_", "/", 1) +} + func isPendingMachineProviderID(providerID normalizedProviderID) bool { return strings.HasPrefix(string(providerID), pendingMachinePrefix) } @@ -610,6 +627,12 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un if found { if providerID != "" { + // If the machine is deleting, prepend the deletion guard on the provider id + // so that it can be properly filtered when counting the number of nodes and instances. + if !machine.GetDeletionTimestamp().IsZero() { + klog.V(4).Infof("Machine %q has a non-zero deletion timestamp", machine.GetName()) + providerID = createDeletingMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName()) + } providerIDs = append(providerIDs, providerID) continue } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index 4e63a7fa640..63800b854f9 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -2131,3 +2131,116 @@ func Test_machineController_nodeGroups(t *testing.T) { }) } } + +func Test_isDeletingMachineProviderID(t *testing.T) { + type testCase struct { + description string + providerID string + expectedReturn bool + } + + testCases := []testCase{ + { + description: "proper provider ID without deletion prefix should return false", + providerID: "fake-provider://a.provider.id-0001", + expectedReturn: false, + }, + { + description: "provider ID with deletion prefix should return true", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + expectedReturn: true, + }, + { + description: "empty provider ID should return false", + providerID: "", + expectedReturn: false, + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID returns true", + providerID: createDeletingMachineNormalizedProviderID("cluster-api", "id-0001"), + expectedReturn: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := isDeletingMachineProviderID(normalizedProviderID(tc.providerID)) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %t, observed %t", tc.providerID, tc.expectedReturn, observed) + } + }) + } + +} + +func Test_machineKeyFromDeletingMachineProviderID(t *testing.T) { + type testCase struct { + description string + providerID string + expectedReturn string + } + + testCases := []testCase{ + { + description: "real looking provider ID with no deletion prefix returns provider id", + providerID: "fake-provider://a.provider.id-0001", + expectedReturn: "fake-provider://a.provider.id-0001", + }, + { + description: "namespace_name provider ID with no deletion prefix returns proper provider ID", + providerID: "cluster-api_id-0001", + expectedReturn: "cluster-api/id-0001", + }, + { + description: "namespace_name provider ID with deletion prefix returns proper provider ID", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + expectedReturn: "cluster-api/id-0001", + }, + { + description: "namespace_name provider ID with deletion prefix and two underscores returns proper provider ID", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id_0001"), + expectedReturn: "cluster-api/id_0001", + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID returns proper provider ID", + providerID: createDeletingMachineNormalizedProviderID("cluster-api", "id-0001"), + expectedReturn: "cluster-api/id-0001", + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := machineKeyFromDeletingMachineProviderID(normalizedProviderID(tc.providerID)) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %q, observed %q", tc.providerID, tc.expectedReturn, observed) + } + }) + } +} + +func Test_createDeletingMachineNormalizedProviderID(t *testing.T) { + type testCase struct { + description string + namespace string + name string + expectedReturn string + } + + testCases := []testCase{ + { + description: "namespace and name return proper normalized ID", + namespace: "cluster-api", + name: "id-0001", + expectedReturn: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := createDeletingMachineNormalizedProviderID(tc.namespace, tc.name) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for (namespace %q, name %q), expected %q, observed %q", tc.namespace, tc.name, tc.expectedReturn, observed) + } + }) + } +} diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index c8204a9c8c6..d79cda3b7e5 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -203,11 +203,28 @@ func (ng *nodegroup) DecreaseTargetSize(delta int) error { return err } - if size+delta < len(nodes) { - return fmt.Errorf("attempt to delete existing nodes targetSize:%d delta:%d existingNodes: %d", - size, delta, len(nodes)) + // we want to filter out machines that are not nodes (eg failed or pending) + // so that the node group size can be set properly. this affects situations + // where an instance is created, but cannot become a node due to cloud provider + // issues such as quota limitations, and thus the autoscaler needs to properly + // set the size of the node group. without this adjustment, the core autoscaler + // will become confused about the state of nodes and instances in the clusterapi + // provider. + actualNodes := 0 + for _, node := range nodes { + if !isPendingMachineProviderID(normalizedProviderID(node.Id)) && + !isFailedMachineProviderID(normalizedProviderID(node.Id)) && + !isDeletingMachineProviderID(normalizedProviderID(node.Id)) { + actualNodes += 1 + } + } + + if size+delta < actualNodes { + return fmt.Errorf("node group %s: attempt to delete existing nodes currentReplicas:%d delta:%d existingNodes: %d", + ng.scalableResource.Name(), size, delta, actualNodes) } + klog.V(4).Infof("%s: DecreaseTargetSize: scaling down: currentReplicas: %d, delta: %d, existingNodes: %d", ng.scalableResource.Name(), size, delta, len(nodes)) return ng.scalableResource.SetSize(size + delta) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index 962ac11f5ed..8e0d40752f7 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -406,18 +406,73 @@ func TestNodeGroupIncreaseSize(t *testing.T) { func TestNodeGroupDecreaseTargetSize(t *testing.T) { type testCase struct { - description string - delta int - initial int32 - targetSizeIncrement int32 - expected int32 - expectedError bool + description string + delta int + initial int32 + targetSizeIncrement int32 + expected int32 + expectedError bool + includeDeletingMachine bool + includeFailedMachine bool + includePendingMachine bool } test := func(t *testing.T, tc *testCase, testConfig *testConfig) { controller, stop := mustCreateTestController(t, testConfig) defer stop() + // machines in deletion should not be counted towards the active nodes when calculating a decrease in size. + if tc.includeDeletingMachine { + if tc.initial < 1 { + t.Fatal("test cannot pass, deleted machine requires at least 1 machine in machineset") + } + + // Simulate a machine in deleting + machine := testConfig.machines[0].DeepCopy() + timestamp := metav1.Now() + machine.SetDeletionTimestamp(×tamp) + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + + // machines that have failed should not be counted towards the active nodes when calculating a decrease in size. + if tc.includeFailedMachine { + // because we want to allow for tests that have deleted machines and failed machines, we use the second machine in the test data. + if tc.initial < 2 { + t.Fatal("test cannot pass, failed machine requires at least 2 machine in machineset") + } + + // Simulate a failed machine + machine := testConfig.machines[1].DeepCopy() + + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + unstructured.SetNestedField(machine.Object, "FailureMessage", "status", "failureMessage") + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + + // machines that are in pending state should not be counted towards the active nodes when calculating a decrease in size. + if tc.includePendingMachine { + // because we want to allow for tests that have deleted, failed machines, and pending machine, we use the third machine in the test data. + if tc.initial < 3 { + t.Fatal("test cannot pass, pending machine requires at least 3 machine in machineset") + } + + // Simulate a pending machine + machine := testConfig.machines[2].DeepCopy() + + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + unstructured.RemoveNestedField(machine.Object, "status", "nodeRef") + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + nodegroups, err := controller.nodeGroups() if err != nil { t.Fatalf("unexpected error: %v", err) @@ -522,45 +577,83 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { } } - annotations := map[string]string{ - nodeGroupMinSizeAnnotationKey: "1", - nodeGroupMaxSizeAnnotationKey: "10", - } - - t.Run("MachineSet", func(t *testing.T) { - tc := testCase{ + testCases := []testCase{ + { description: "Same number of existing instances and node group target size should error", initial: 3, targetSizeIncrement: 0, expected: 3, delta: -1, expectedError: true, - } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) - - t.Run("MachineSet", func(t *testing.T) { - tc := testCase{ + }, + { description: "A node group with target size 4 but only 3 existing instances should decrease by 1", initial: 3, targetSizeIncrement: 1, expected: 3, delta: -1, - } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) + }, + { + description: "A node group with 4 replicas with one machine in deleting state should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeDeletingMachine: true, + }, + { + description: "A node group with 4 replicas with one failed machine should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeFailedMachine: true, + }, + { + description: "A node group with 4 replicas with one pending machine should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includePendingMachine: true, + }, + { + description: "A node group with 5 replicas with one pending and one failed machine should decrease by 2", + initial: 5, + targetSizeIncrement: 0, + expected: 3, + delta: -2, + includeFailedMachine: true, + includePendingMachine: true, + }, + { + description: "A node group with 5 replicas with one pending, one failed, and one deleting machine should decrease by 3", + initial: 5, + targetSizeIncrement: 0, + expected: 2, + delta: -3, + includeFailedMachine: true, + includePendingMachine: true, + includeDeletingMachine: true, + }, + } - t.Run("MachineDeployment", func(t *testing.T) { - tc := testCase{ - description: "Same number of existing instances and node group target size should error", - initial: 3, - targetSizeIncrement: 0, - expected: 3, - delta: -1, - expectedError: true, - } - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) + annotations := map[string]string{ + nodeGroupMinSizeAnnotationKey: "1", + nodeGroupMaxSizeAnnotationKey: "10", + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + }) + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + }) + } } func TestNodeGroupDecreaseSizeErrors(t *testing.T) { @@ -580,7 +673,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) { description: "errors because initial+delta < len(nodes)", delta: -1, initial: 3, - errorMsg: "attempt to delete existing nodes targetSize:3 delta:-1 existingNodes: 3", + errorMsg: "attempt to delete existing nodes currentReplicas:3 delta:-1 existingNodes: 3", }} test := func(t *testing.T, tc *testCase, testConfig *testConfig) { From 9043687eb7027157181e6d3e48f1df9d83f67163 Mon Sep 17 00:00:00 2001 From: Marco Voelz Date: Tue, 18 Mar 2025 09:25:06 +0100 Subject: [PATCH 15/39] Fix typo --- .../pkg/recommender/input/cluster_feeder_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index aaffed0548d..7705cb64212 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -420,13 +420,13 @@ func makeTestSpecClient(podLabels []map[string]string) spec.SpecClient { } } -func newTestContainerSpec(podID model.PodID, containerName string, milicores int, memory int64) spec.BasicContainerSpec { +func newTestContainerSpec(podID model.PodID, containerName string, millicores int, memory int64) spec.BasicContainerSpec { containerID := model.ContainerID{ PodID: podID, ContainerName: containerName, } requestedResources := model.Resources{ - model.ResourceCPU: model.ResourceAmount(milicores), + model.ResourceCPU: model.ResourceAmount(millicores), model.ResourceMemory: model.ResourceAmount(memory), } return spec.BasicContainerSpec{ From e34783615d95519cf19322b4421625a345a0d511 Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Tue, 18 Mar 2025 10:48:20 -0400 Subject: [PATCH 16/39] Address comments + add a feature enablement/rollback section --- .../7862-cpu-startup-boost/README.md | 100 ++++++++++++------ 1 file changed, 69 insertions(+), 31 deletions(-) diff --git a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md index 261d62b989e..76689d6182e 100644 --- a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md +++ b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md @@ -13,8 +13,10 @@ - [Validation](#validation) - [Static Validation](#static-validation) - [Dynamic Validation](#dynamic-validation) - - [Feature Enablement](#feature-enablement) - [Mitigating Failed In-Place Downsizes](#mitigating-failed-in-place-downsizes) + - [Feature Enablement and Rollback](#feature-enablement-and-rollback) + - [How can this feature be enabled / disabled in a live cluster?](#how-can-this-feature-be-enabled--disabled-in-a-live-cluster) + - [Kubernetes Version Compatibility](#kubernetes-version-compatibility) - [Test Plan](#test-plan) - [Examples](#examples) - [CPU Boost Only](#cpu-boost-only) @@ -35,6 +37,9 @@ This proposal allows VPA to boost the CPU request and limit of containers during the pod startup and to scale the CPU resources back down when the pod is `Ready`, leveraging the [in-place pod resize Kubernetes feature](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). +> [!NOTE] +> This feature depends on the new `InPlaceOrRecreate` VPA mode: +> [AEP-4016: Support for in place updates in VPA](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md) ### Goals @@ -92,11 +97,20 @@ and contain the following fields: * `StartupBoost.CPU.Value`: the target value of the CPU request or limit during the startup boost phase. -NOTE: The boosted CPU value will be capped by -[`--container-recommendation-max-allowed-cpu`](https://github.com/kubernetes/autoscaler/blob/4d294562e505431d518a81e8833accc0ec99c9b8/vertical-pod-autoscaler/pkg/recommender/main.go#L122) -flag value, if set. +> [!IMPORTANT] +> The boosted CPU value will be capped by +> [`--container-recommendation-max-allowed-cpu`](https://github.com/kubernetes/autoscaler/blob/4d294562e505431d518a81e8833accc0ec99c9b8/vertical-pod-autoscaler/pkg/recommender/main.go#L122) +> flag value, if set. -NOTE: Only one of `Factor` or `Value` may be specified per container policy. +> [!IMPORTANT] +> Only one of `Factor` or `Value` may be specified per container policy. + +We will also add a new mode to the [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236): + * **NEW**: `StartupBoostOnly`: new mode that will allow users to only enable + the startup boost feature for a container and not vanilla VPA altogether. + * **NEW**: `Auto`: we will modify the existing `Auto` mode to enable both + vanilla VPA and CPU Startup Boost (when `StartupBoost` parameter is + specified). #### Priority of `StartupBoost` @@ -121,9 +135,11 @@ excluded from [`ControlledResources`](https://github.com/kubernetes/autoscaler/b are created/updated: * The VPA autoscaling mode must be `InPlaceOrRecreate` (since it does not make sense to use this feature with disruptive modes of VPA). - * The boost factor is >= 1 + * The boost factor is >= 1 (via CRD validation rules) * Only one of `StartupBoost.CPU.Factor` or `StartupBoost.CPU.Value` is specified + * The [feature enablement](#feature-enablement) flags must be on. + #### Dynamic Validation @@ -134,35 +150,58 @@ are created/updated: utilize this feature. Therefore, VPA will not boost CPU resources of workloads that do not configure a Readiness or a Startup probe. -### Feature Enablement - -During the Alpha launch of this feature, users will need to: -* Enable the CPU startup boost feature via a binary flag. -* Set the VPA autoscaling mode to `InPlaceOrRecreate` -* Set the [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) -to either: - * **NEW**: `StartupBoostOnly`: this mode will allow users to only enable the - startup boost feature for a container and not vanilla VPA altogether. - * **NEW**: `Auto`: in this mode, both vanilla VPA and CPU Startup Boost will - be enabled. - - ### Mitigating Failed In-Place Downsizes -The VPA Updater will evict a pod to actuate a startup CPU boost +The VPA Updater **will not** evict a pod to actuate a startup CPU boost recommendation if it attempted to apply the recommendation in place and it -failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169) -where the VPA updater will consider that the update failed). - -We will cache pods that have failed the in-place downsize and not reattempt to -CPU boost them again for the next hour, to **avoid** an eviction loop scenario -like this: +failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA +updater will consider that the update failed). This is to avoid an eviction +loop: 1. A pod is created and has its CPU resources boosted -2. The pod is ready. VPA Updater tries to downscale the pod in-place and it +1. The pod is ready. VPA Updater tries to downscale the pod in-place and it fails. -1. VPA Updater evicts the pod. (If we do nothing, the logic flow goes back to -(1)). +1. VPA Updater evicts the pod. Logic flow goes back to (1). + +### Feature Enablement and Rollback + +#### How can this feature be enabled / disabled in a live cluster? + +* Feature gates names: `CPUStartupBoost` and `InPlaceOrRecreate` (from +[AEP-4016](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md#feature-enablement-and-rollback)) +* Components depending on the feature gates: + * admission-controller + * updater + +Enabling of feature gates `CPUStartupBoost` AND `InPlaceOrRecreate` will cause +the following to happen: + * admission-controller to **accept** new VPA objects being created with +`StartupBoostOnly` configured. + * admission-controller to **boost** CPU resources. + * updater to **unboost** the CPU resources. + +Disabling of feature gates `CPUStartupBoost` OR `InPlaceOrRecreate` will cause +the following to happen: + * admission-controller to **reject** new VPA objects being created with + `StartupBoostOnly` configured. + * A descriptive error message should be returned to the user letting them + know that they are using a feature gated feature. + * admission-controller **to not** boost CPU resources, should it encounter a + VPA configured with a `StartupBoost` config and `StartupBoostOnly` or `Auto` + `ContainerScalingMode`. + * updater **to not** unboost CPU resources when pods become `Ready`, should it + encounter a VPA configured with a `StartupBoost` config and `StartupBoostOnly` + or `Auto` `ContainerScalingMode`. + +### Kubernetes Version Compatibility + +Similarly to [AEP-4016](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support#kubernetes-version-compatibility), +`StartupBoost` configuration and `StartupBoostOnly` mode are built assuming that +VPA will be running on a Kubernetes 1.33+ with the beta version of +[KEP-1287: In-Place Update of Pod Resources](https://github.com/kubernetes/enhancements/issues/1287) +enabled. If this is not the case, VPA will behave as if the `CPUStartupBoost` +feature gate was disabled (see [Feature Enablement and Rollback](#feature-enablement-and-rollback) +section for details). ## Test Plan @@ -178,7 +217,6 @@ Readiness or a Startup probe. * Pod is evicted the first time that an in-place update fails when scaling the pod back down. And a new CPU boost is not attempted when the pod is recreated. - ## Examples Here are some examples of the VPA CR incorporating CPU boosting for different @@ -241,5 +279,5 @@ spec: ## Implementation History -* 2025-03-12: Initial version. +* 2025-03-18: Initial version. From 9a5e3d9f3dc5de7e4291890c77f15b4203afaa66 Mon Sep 17 00:00:00 2001 From: Norbert Cyran Date: Wed, 19 Mar 2025 06:43:11 +0100 Subject: [PATCH 17/39] Allow using scheduled pods as samples in proactive scale up --- .../processors/podinjection/pod_group.go | 2 +- .../podinjection/pod_injection_processor.go | 1 + .../pod_injection_processor_test.go | 154 ++++++++++-------- cluster-autoscaler/utils/test/test_utils.go | 15 +- 4 files changed, 100 insertions(+), 72 deletions(-) diff --git a/cluster-autoscaler/processors/podinjection/pod_group.go b/cluster-autoscaler/processors/podinjection/pod_group.go index 747e05a55f2..fa4c4bccfa6 100644 --- a/cluster-autoscaler/processors/podinjection/pod_group.go +++ b/cluster-autoscaler/processors/podinjection/pod_group.go @@ -57,7 +57,7 @@ func updatePodGroups(pod *apiv1.Pod, ownerRef metav1.OwnerReference, podGroups m if !found { return podGroups } - if group.sample == nil && pod.Spec.NodeName == "" { + if group.sample == nil || pod.CreationTimestamp.After(group.sample.CreationTimestamp.Time) { group.sample = pod group.ownerUid = ownerRef.UID } diff --git a/cluster-autoscaler/processors/podinjection/pod_injection_processor.go b/cluster-autoscaler/processors/podinjection/pod_injection_processor.go index 08645fa4d1d..82b44e0171f 100644 --- a/cluster-autoscaler/processors/podinjection/pod_injection_processor.go +++ b/cluster-autoscaler/processors/podinjection/pod_injection_processor.go @@ -93,6 +93,7 @@ func makeFakePods(ownerUid types.UID, samplePod *apiv1.Pod, podCount int) []*api newPod := withFakePodAnnotation(samplePod.DeepCopy()) newPod.Name = fmt.Sprintf("%s-copy-%d", samplePod.Name, i) newPod.UID = types.UID(fmt.Sprintf("%s-%d", string(ownerUid), i)) + newPod.Spec.NodeName = "" fakePods = append(fakePods, newPod) } return fakePods diff --git a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go index 4ce6b895b15..d709757255b 100644 --- a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go +++ b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go @@ -19,6 +19,7 @@ package podinjection import ( "fmt" "testing" + "time" "github.com/stretchr/testify/assert" @@ -40,24 +41,24 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { node := BuildTestNode("node1", 100, 0) replicaSet1 := createTestReplicaSet("rep-set-1", "default", 5) - scheduledPodRep1Copy1 := buildTestPod("default", "-scheduled-pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName(node.Name)) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + scheduledPodRep1Copy1 := buildTestPod("default", "-scheduled-pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName(node.Name)) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) job1 := createTestJob("job-1", "default", 10, 10, 0) - scheduledPodJob1Copy1 := buildTestPod("default", "scheduled-pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID), withNodeName(node.Name)) - podJob1Copy1 := buildTestPod("default", "pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID)) - podJob1Copy2 := buildTestPod("default", "pod-job1-2", withControllerOwnerRef(job1.Name, "Job", job1.UID)) + scheduledPodJob1Copy1 := buildTestPod("default", "scheduled-pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID), WithNodeName(node.Name)) + podJob1Copy1 := buildTestPod("default", "pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy2 := buildTestPod("default", "pod-job1-2", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) parallelStatefulset := createTestStatefulset("parallel-statefulset-1", "default", appsv1.ParallelPodManagement, 10) - scheduledParallelStatefulsetPod := buildTestPod("default", "parallel-scheduled-pod-statefulset-1", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID), withNodeName(node.Name)) - parallelStatefulsetPodCopy1 := buildTestPod("default", "parallel-pod-statefulset1-1", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) - parallelStatefulsetPodCopy2 := buildTestPod("default", "parallel-pod-statefulset1-2", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) + scheduledParallelStatefulsetPod := buildTestPod("default", "parallel-scheduled-pod-statefulset-1", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID), WithNodeName(node.Name)) + parallelStatefulsetPodCopy1 := buildTestPod("default", "parallel-pod-statefulset1-1", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) + parallelStatefulsetPodCopy2 := buildTestPod("default", "parallel-pod-statefulset1-2", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) sequentialStatefulset := createTestStatefulset("sequential-statefulset-1", "default", appsv1.OrderedReadyPodManagement, 10) - scheduledSequentialStatefulsetPod := buildTestPod("default", "sequential-scheduled-pod-statefulset-1", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID), withNodeName(node.Name)) - sequentialStatefulsetPodCopy1 := buildTestPod("default", "sequential-pod-statefulset1-1", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) - sequentialStatefulsetPodCopy2 := buildTestPod("default", "sequential-pod-statefulset1-2", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) + scheduledSequentialStatefulsetPod := buildTestPod("default", "sequential-scheduled-pod-statefulset-1", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID), WithNodeName(node.Name)) + sequentialStatefulsetPodCopy1 := buildTestPod("default", "sequential-pod-statefulset1-1", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) + sequentialStatefulsetPodCopy2 := buildTestPod("default", "sequential-pod-statefulset1-2", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) replicaSetLister, err := kubernetes.NewTestReplicaSetLister([]*appsv1.ReplicaSet{&replicaSet1}) assert.NoError(t, err) @@ -67,46 +68,46 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { assert.NoError(t, err) testCases := []struct { - name string - scheduledPods []*apiv1.Pod - unschedulabePods []*apiv1.Pod - wantPods []*apiv1.Pod + name string + scheduledPods []*apiv1.Pod + unschedulablePods []*apiv1.Pod + wantPods []*apiv1.Pod }{ { - name: "ReplicaSet", - scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1}, - unschedulabePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2}, - wantPods: append([]*apiv1.Pod{podRep1Copy1, podRep1Copy2}, makeFakePods(replicaSet1.UID, podRep1Copy1, 2)...), + name: "ReplicaSet", + scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1}, + unschedulablePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2}, + wantPods: append([]*apiv1.Pod{podRep1Copy1, podRep1Copy2}, makeFakePods(replicaSet1.UID, scheduledPodRep1Copy1, 2)...), }, { - name: "Job", - scheduledPods: []*apiv1.Pod{scheduledPodJob1Copy1}, - unschedulabePods: []*apiv1.Pod{podJob1Copy1, podJob1Copy2}, - wantPods: append([]*apiv1.Pod{podJob1Copy1, podJob1Copy2}, makeFakePods(job1.UID, podJob1Copy1, 7)...), + name: "Job", + scheduledPods: []*apiv1.Pod{scheduledPodJob1Copy1}, + unschedulablePods: []*apiv1.Pod{podJob1Copy1, podJob1Copy2}, + wantPods: append([]*apiv1.Pod{podJob1Copy1, podJob1Copy2}, makeFakePods(job1.UID, scheduledPodJob1Copy1, 7)...), }, { - name: "Statefulset - Parallel pod management policy", - scheduledPods: []*apiv1.Pod{scheduledParallelStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, - wantPods: append([]*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, makeFakePods(parallelStatefulset.UID, parallelStatefulsetPodCopy1, 7)...), + name: "Statefulset - Parallel pod management policy", + scheduledPods: []*apiv1.Pod{scheduledParallelStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, + wantPods: append([]*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, makeFakePods(parallelStatefulset.UID, scheduledParallelStatefulsetPod, 7)...), }, { - name: "Statefulset - sequential pod management policy", - scheduledPods: []*apiv1.Pod{scheduledSequentialStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, - wantPods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, + name: "Statefulset - sequential pod management policy", + scheduledPods: []*apiv1.Pod{scheduledSequentialStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, + wantPods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, }, { - name: "Mix of controllers", - scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1, scheduledPodJob1Copy1, scheduledParallelStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, + name: "Mix of controllers", + scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1, scheduledPodJob1Copy1, scheduledParallelStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, wantPods: append( append( append( []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, - makeFakePods(replicaSet1.UID, podRep1Copy1, 2)...), - makeFakePods(job1.UID, podJob1Copy1, 7)...), - makeFakePods(parallelStatefulset.UID, parallelStatefulsetPodCopy1, 7)..., + makeFakePods(replicaSet1.UID, scheduledPodRep1Copy1, 2)...), + makeFakePods(job1.UID, scheduledPodJob1Copy1, 7)...), + makeFakePods(parallelStatefulset.UID, scheduledParallelStatefulsetPod, 7)..., ), }, } @@ -123,7 +124,7 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { }, ClusterSnapshot: clusterSnapshot, } - pods, err := p.Process(&ctx, tc.unschedulabePods) + pods, err := p.Process(&ctx, tc.unschedulablePods) assert.NoError(t, err) assert.ElementsMatch(t, tc.wantPods, pods) }) @@ -134,31 +135,46 @@ func TestGroupPods(t *testing.T) { noControllerPod := buildTestPod("default", "pod-no-podGroup") replicaSet1 := createTestReplicaSet("rep-set-1", "default", 10) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1ScheduledCopy1 := buildTestPod("default", "pod-rep1-3", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName("n1")) - podRep1ScheduledCopy2 := buildTestPod("default", "pod-rep1-4", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName("n1")) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1ScheduledCopy1 := buildTestPod("default", "pod-rep1-3", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName("n1")) + podRep1ScheduledCopy2 := buildTestPod("default", "pod-rep1-4", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName("n1")) + timestamp := time.Date(2025, 3, 19, 12, 0, 0, 0, time.UTC) + podRep1OlderCopy := buildTestPod( + "default", + "pod-rep1-5", + WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), + WithNodeName("n1"), + WithCreationTimestamp(timestamp), + ) + podRep1NewerCopy := buildTestPod( + "default", + "pod-rep1-6", + WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), + WithNodeName("n1"), + WithCreationTimestamp(timestamp.Add(1*time.Minute)), + ) replicaSet2 := createTestReplicaSet("rep-set-2", "default", 10) - podRep2Copy1 := buildTestPod("default", "pod-rep2-1", withControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID)) - podRep2ScheduledCopy1 := buildTestPod("default", "pod-rep2-1", withControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID), withNodeName("n1")) + podRep2Copy1 := buildTestPod("default", "pod-rep2-1", WithControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID)) + podRep2ScheduledCopy1 := buildTestPod("default", "pod-rep2-1", WithControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID), WithNodeName("n1")) replicaSet3 := createTestReplicaSet("rep-set-3", "default", 10) - podRep3Copy1 := buildTestPod("default", "pod-rep3-1", withControllerOwnerRef(replicaSet3.Name, "ReplicaSet", replicaSet3.UID)) + podRep3Copy1 := buildTestPod("default", "pod-rep3-1", WithControllerOwnerRef(replicaSet3.Name, "ReplicaSet", replicaSet3.UID)) job1 := createTestJob("job-1", "default", 10, 10, 0) - podJob1Copy1 := buildTestPod("default", "pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID)) - podJob1Copy2 := buildTestPod("default", "pod-job1-2", withControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy1 := buildTestPod("default", "pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy2 := buildTestPod("default", "pod-job1-2", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) job2 := createTestJob("job-2", "default", 10, 10, 0) - podJob2Copy1 := buildTestPod("default", "pod-job-2", withControllerOwnerRef(job2.Name, "Job", job2.UID)) + podJob2Copy1 := buildTestPod("default", "pod-job-2", WithControllerOwnerRef(job2.Name, "Job", job2.UID)) statefulset1 := createTestStatefulset("statefulset-1", "default", appsv1.ParallelPodManagement, 10) - statefulset1Copy1 := buildTestPod("default", "pod-statefulset1-1", withControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) - statefulset1Copy2 := buildTestPod("default", "pod-statefulset1-2", withControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) + statefulset1Copy1 := buildTestPod("default", "pod-statefulset1-1", WithControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) + statefulset1Copy2 := buildTestPod("default", "pod-statefulset1-2", WithControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) statefulset2 := createTestStatefulset("statefulset-2", "default", appsv1.ParallelPodManagement, 10) - statefulset2Copy1 := buildTestPod("default", "pod-statefulset2-1", withControllerOwnerRef(statefulset2.Name, "StatefulSet", statefulset2.UID)) + statefulset2Copy1 := buildTestPod("default", "pod-statefulset2-1", WithControllerOwnerRef(statefulset2.Name, "StatefulSet", statefulset2.UID)) testCases := []struct { name string @@ -182,8 +198,8 @@ func TestGroupPods(t *testing.T) { scheduledPods: []*apiv1.Pod{podRep1ScheduledCopy1, podRep1ScheduledCopy2, podRep2ScheduledCopy1}, replicaSets: []*appsv1.ReplicaSet{&replicaSet1, &replicaSet2}, wantGroupedPods: map[types.UID]podGroup{ - replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: nil}, - replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: nil}, + replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1ScheduledCopy1, ownerUid: replicaSet1.UID}, + replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: podRep2ScheduledCopy1, ownerUid: replicaSet2.UID}, }, }, { @@ -192,7 +208,7 @@ func TestGroupPods(t *testing.T) { unscheduledPods: []*apiv1.Pod{podRep1Copy1, podRep2Copy1}, replicaSets: []*appsv1.ReplicaSet{&replicaSet1, &replicaSet2}, wantGroupedPods: map[types.UID]podGroup{ - replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1Copy1, ownerUid: replicaSet1.UID}, + replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1ScheduledCopy2, ownerUid: replicaSet1.UID}, replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: podRep2Copy1, ownerUid: replicaSet2.UID}, }, }, @@ -246,6 +262,15 @@ func TestGroupPods(t *testing.T) { statefulset1.UID: {podCount: 1, desiredReplicas: 10, sample: statefulset1Copy1, ownerUid: statefulset1.UID}, }, }, + { + name: "newest pod as a sample", + unscheduledPods: []*apiv1.Pod{podRep1Copy1}, + scheduledPods: []*apiv1.Pod{podRep1NewerCopy, podRep1OlderCopy}, + replicaSets: []*appsv1.ReplicaSet{&replicaSet1}, + wantGroupedPods: map[types.UID]podGroup{ + replicaSet1.UID: {podCount: 3, desiredReplicas: 10, sample: podRep1NewerCopy, ownerUid: replicaSet1.UID}, + }, + }, } for _, tc := range testCases { @@ -271,8 +296,8 @@ func TestGroupPods(t *testing.T) { func TestUpdatePodGroups(t *testing.T) { replicaSet1 := createTestReplicaSet("rep-set-1", "default", 10) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) samplePodGroups := map[types.UID]podGroup{replicaSet1.UID: makePodGroup(10)} sampleFalse := false sampleTrue := true @@ -347,7 +372,7 @@ func TestUpdatePodGroups(t *testing.T) { } } func TestMakeFakePods(t *testing.T) { - samplePod := buildTestPod("default", "test-pod") + samplePod := buildTestPod("default", "test-pod", WithNodeName("test-node")) // Test case: Positive fake pod count fakePodCount := 5 ownerUid := types.UID("sample uid") @@ -356,6 +381,7 @@ func TestMakeFakePods(t *testing.T) { for idx, fakePod := range fakePods { assert.Equal(t, fakePod.Name, fmt.Sprintf("%s-copy-%d", samplePod.Name, idx+1)) assert.Equal(t, fakePod.UID, types.UID(fmt.Sprintf("%s-%d", string(ownerUid), idx+1))) + assert.Equal(t, "", fakePod.Spec.NodeName) assert.NotNil(t, fakePod.Annotations) assert.Equal(t, fakePod.Annotations[FakePodAnnotationKey], FakePodAnnotationValue) } @@ -407,15 +433,3 @@ func buildTestPod(namespace, name string, opts ...podOption) *apiv1.Pod { } type podOption func(*apiv1.Pod) - -func withControllerOwnerRef(name, kind string, uid types.UID) podOption { - return func(pod *apiv1.Pod) { - pod.OwnerReferences = GenerateOwnerReferences(name, kind, "apps/v1", uid) - } -} - -func withNodeName(nodeName string) podOption { - return func(pod *apiv1.Pod) { - pod.Spec.NodeName = nodeName - } -} diff --git a/cluster-autoscaler/utils/test/test_utils.go b/cluster-autoscaler/utils/test/test_utils.go index 577007744b8..0b9bfa7aafc 100644 --- a/cluster-autoscaler/utils/test/test_utils.go +++ b/cluster-autoscaler/utils/test/test_utils.go @@ -26,7 +26,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/mock" - apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -113,6 +112,13 @@ func WithResourceClaim(refName, claimName, templateName string) func(*apiv1.Pod) } } +// WithControllerOwnerRef sets an owner reference to the pod. +func WithControllerOwnerRef(name, kind string, uid types.UID) func(*apiv1.Pod) { + return func(pod *apiv1.Pod) { + pod.OwnerReferences = GenerateOwnerReferences(name, kind, "apps/v1", uid) + } +} + // WithDSController creates a daemonSet owner ref for the pod. func WithDSController() func(*apiv1.Pod) { return func(pod *apiv1.Pod) { @@ -174,6 +180,13 @@ func WithMaxSkew(maxSkew int32, topologySpreadingKey string) func(*apiv1.Pod) { } } +// WithCreationTimestamp sets creation timestamp to the pod. +func WithCreationTimestamp(timestamp time.Time) func(*apiv1.Pod) { + return func(pod *apiv1.Pod) { + pod.CreationTimestamp = metav1.Time{Time: timestamp} + } +} + // WithDeletionTimestamp sets deletion timestamp to the pod. func WithDeletionTimestamp(deletionTimestamp time.Time) func(*apiv1.Pod) { return func(pod *apiv1.Pod) { From 105429c31e8c10faf5a737bfa0b76b5f877d81ef Mon Sep 17 00:00:00 2001 From: Yuriy Stryuchkov <43517125+ystryuchkov@users.noreply.github.com> Date: Wed, 19 Mar 2025 15:49:34 +0100 Subject: [PATCH 18/39] Fix log for node filtering in static autoscaler Add missing tests --- cluster-autoscaler/core/static_autoscaler.go | 2 +- .../core/static_autoscaler_test.go | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index 68992602383..26e2d58a3f8 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -1001,7 +1001,7 @@ func filterNodesFromSelectedGroups(cp cloudprovider.CloudProvider, nodes ...*api filtered := make([]*apiv1.Node, 0, len(nodes)) for _, n := range nodes { if ng, err := cp.NodeGroupForNode(n); err != nil { - klog.Errorf("Failed to get a node group node node: %v", err) + klog.Errorf("Failed to get a nodegroup for node %q: %v", n.Name, err) } else if ng != nil { filtered = append(filtered, n) } diff --git a/cluster-autoscaler/core/static_autoscaler_test.go b/cluster-autoscaler/core/static_autoscaler_test.go index ec0e549dffd..d30536e0cdd 100644 --- a/cluster-autoscaler/core/static_autoscaler_test.go +++ b/cluster-autoscaler/core/static_autoscaler_test.go @@ -2674,6 +2674,65 @@ func TestStaticAutoscalerRunOnceInvokesScaleDownStatusProcessor(t *testing.T) { } +func TestFilterNodesFromSelectedGroups(t *testing.T) { + node1 := BuildTestNode("node1", 1000, 1000) + node1.Spec.ProviderID = "A" + node2 := BuildTestNode("node2", 1000, 1000) + node2.Spec.ProviderID = "B" + node3 := BuildTestNode("node3", 1000, 1000) + node3.Spec.ProviderID = "C" + invalidNode := BuildTestNode("invalidNode", 1000, 1000) + invalidNode.Spec.ProviderID = "invalid" + + provider := &mockprovider.CloudProvider{} + provider.On("NodeGroupForNode", mock.Anything).Return( + func(node *apiv1.Node) cloudprovider.NodeGroup { + if node.Spec.ProviderID == "A" || node.Spec.ProviderID == "B" { + return &mockprovider.NodeGroup{} + } + return nil + }, func(node *apiv1.Node) error { + if node.Spec.ProviderID == "invalid" { + return fmt.Errorf("broken provider") + } + return nil + }) + + tests := []struct { + name string + nodes []*apiv1.Node + wantNodes []*apiv1.Node + }{ + { + name: "returns no nodes if none were provided", + nodes: []*apiv1.Node{}, + wantNodes: []*apiv1.Node{}, + }, + { + name: "returns nodes with matching providers", + nodes: []*apiv1.Node{node1, node2}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + { + name: "filters out nodes with not matching provider", + nodes: []*apiv1.Node{node1, node2, node3}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + { + name: "filters out nodes with broken provider", + nodes: []*apiv1.Node{node1, node2, invalidNode}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filteredNodes := filterNodesFromSelectedGroups(provider, tt.nodes...) + assert.Equal(t, tt.wantNodes, filteredNodes) + }) + } +} + func waitForDeleteToFinish(t *testing.T, deleteFinished <-chan bool) { t.Helper() select { From 71d3595cb7ccd6fedb31d6c603465639ce7ef540 Mon Sep 17 00:00:00 2001 From: elmiko Date: Wed, 19 Mar 2025 10:54:49 -0400 Subject: [PATCH 19/39] improve failed machine detection in clusterapi This change makes it so that when a failed machine is found during the `findScalableResourceProviderIDs` it will always gain a normalized provider ID with failure guard prepended. This is to ensure that machines which have gained a provider ID from the infrastructure and then later go into a failed state can be properly removed by the autoscaler when it wants to correct the size of a node group. --- .../clusterapi/clusterapi_controller.go | 46 ++++++++++++------- .../clusterapi/clusterapi_controller_test.go | 10 ++++ .../clusterapi/clusterapi_nodegroup_test.go | 32 +++++++++---- 3 files changed, 62 insertions(+), 26 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index ab2095ef594..3a4e7c75927 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -365,6 +365,10 @@ func machineKeyFromPendingMachineProviderID(providerID normalizedProviderID) str return strings.Replace(namespaceName, "_", "/", 1) } +func createFailedMachineNormalizedProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", failedMachinePrefix, namespace, name) +} + func isFailedMachineProviderID(providerID normalizedProviderID) bool { return strings.HasPrefix(string(providerID), failedMachinePrefix) } @@ -620,6 +624,30 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un } for _, machine := range machines { + // In some cases it is possible for a machine to have acquired a provider ID from the infrastructure and + // then become failed later. We want to ensure that a failed machine is not counted towards the total + // number of nodes in the cluster, for this reason we will detect a failed machine first, regardless + // of provider ID, and give it a normalized provider ID with failure message prepended. + failureMessage, found, err := unstructured.NestedString(machine.UnstructuredContent(), "status", "failureMessage") + if err != nil { + return nil, err + } + + if found { + klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) + // Provide a fake ID to allow the autoscaler to track machines that will never + // become nodes and mark the nodegroup unhealthy after maxNodeProvisionTime. + // Fake ID needs to be recognised later and converted into a machine key. + // Use an underscore as a separator between namespace and name as it is not a + // valid character within a namespace name. + providerIDs = append(providerIDs, createFailedMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName())) + continue + } + + // Next we check for the provider ID. Machines with a provider ID can fall into one of a few + // categories: creating, running, deleting, and sometimes failed (but we filtered those earlier). + // Depending on which category the machine is in, it might have a deleting or pending guard + // added to it's provider ID so that we can later properly filter machines. providerID, found, err := unstructured.NestedString(machine.UnstructuredContent(), "spec", "providerID") if err != nil { return nil, err @@ -640,22 +668,8 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un klog.Warningf("Machine %q has no providerID", machine.GetName()) - failureMessage, found, err := unstructured.NestedString(machine.UnstructuredContent(), "status", "failureMessage") - if err != nil { - return nil, err - } - - if found { - klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) - // Provide a fake ID to allow the autoscaler to track machines that will never - // become nodes and mark the nodegroup unhealthy after maxNodeProvisionTime. - // Fake ID needs to be recognised later and converted into a machine key. - // Use an underscore as a separator between namespace and name as it is not a - // valid character within a namespace name. - providerIDs = append(providerIDs, fmt.Sprintf("%s%s_%s", failedMachinePrefix, machine.GetNamespace(), machine.GetName())) - continue - } - + // Look for a node reference in the status, a machine with a provider ID but no node reference has not + // yet become a node and should be marked as pending. _, found, err = unstructured.NestedFieldCopy(machine.UnstructuredContent(), "status", "nodeRef") if err != nil { return nil, err diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index 63800b854f9..ecedc7c032a 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -1664,6 +1664,11 @@ func TestIsFailedMachineProviderID(t *testing.T) { providerID: normalizedProviderID("foo"), expected: false, }, + { + name: "with provider ID created by createFailedMachineNormalizedProviderID should return true", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expected: true, + }, } for _, tc := range testCases { @@ -1691,6 +1696,11 @@ func TestMachineKeyFromFailedProviderID(t *testing.T) { providerID: normalizedProviderID(fmt.Sprintf("%stest-namespace_foo_bar", failedMachinePrefix)), expected: "test-namespace/foo_bar", }, + { + name: "with a provider ID created by createFailedMachineNormalizedProviderID", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expected: "cluster-api/id-0001", + }, } for _, tc := range testCases { diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index 8e0d40752f7..11bcec4e3df 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -406,15 +406,16 @@ func TestNodeGroupIncreaseSize(t *testing.T) { func TestNodeGroupDecreaseTargetSize(t *testing.T) { type testCase struct { - description string - delta int - initial int32 - targetSizeIncrement int32 - expected int32 - expectedError bool - includeDeletingMachine bool - includeFailedMachine bool - includePendingMachine bool + description string + delta int + initial int32 + targetSizeIncrement int32 + expected int32 + expectedError bool + includeDeletingMachine bool + includeFailedMachine bool + includeFailedMachineWithProviderID bool + includePendingMachine bool } test := func(t *testing.T, tc *testCase, testConfig *testConfig) { @@ -447,7 +448,9 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { // Simulate a failed machine machine := testConfig.machines[1].DeepCopy() - unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + if !tc.includeFailedMachineWithProviderID { + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + } unstructured.SetNestedField(machine.Object, "FailureMessage", "status", "failureMessage") if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { @@ -636,6 +639,15 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { includePendingMachine: true, includeDeletingMachine: true, }, + { + description: "A node group with 4 replicas with one failed machine that has a provider ID should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeFailedMachine: true, + includeFailedMachineWithProviderID: true, + }, } annotations := map[string]string{ From 4aa465764c8cc341975c90dcb342c2c4e13aa870 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 19 Mar 2025 11:40:19 -0700 Subject: [PATCH 20/39] capi: node and provider ID accounting funcs Signed-off-by: Jack Francis --- .../clusterapi/clusterapi_controller.go | 16 ++++- .../clusterapi/clusterapi_controller_test.go | 71 +++++++++++++++++++ .../clusterapi/clusterapi_nodegroup.go | 4 +- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index 3a4e7c75927..76ed9051c51 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -356,6 +356,11 @@ func machineKeyFromDeletingMachineProviderID(providerID normalizedProviderID) st return strings.Replace(namespaceName, "_", "/", 1) } +// createPendingMachineProviderID creates a providerID for a machine that is pending +func createPendingMachineProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", pendingMachinePrefix, namespace, name) +} + func isPendingMachineProviderID(providerID normalizedProviderID) bool { return strings.HasPrefix(string(providerID), pendingMachinePrefix) } @@ -378,6 +383,15 @@ func machineKeyFromFailedProviderID(providerID normalizedProviderID) string { return strings.Replace(namespaceName, "_", "/", 1) } +// nodeHasValidProviderID determines whether a node's providerID is the standard +// providerID assigned by the cloud provider, or if it has +// been modified by the CAS CAPI provider to indicate deleting, pending, or failed +func nodeHasValidProviderID(providerID normalizedProviderID) bool { + return !isDeletingMachineProviderID(providerID) && + !isPendingMachineProviderID(providerID) && + !isFailedMachineProviderID(providerID) +} + // findNodeByNodeName finds the Node object keyed by name.. Returns // nil if it cannot be found. A DeepCopy() of the object is returned // on success. @@ -677,7 +691,7 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un if !found { klog.V(4).Infof("Status.NodeRef of machine %q is currently nil", machine.GetName()) - providerIDs = append(providerIDs, fmt.Sprintf("%s%s_%s", pendingMachinePrefix, machine.GetNamespace(), machine.GetName())) + providerIDs = append(providerIDs, createPendingMachineProviderID(machine.GetNamespace(), machine.GetName())) continue } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index ecedc7c032a..a468170bea0 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -2183,6 +2183,49 @@ func Test_isDeletingMachineProviderID(t *testing.T) { } +// TestNodeHasValidProviderID tests all permutations of provider IDs +// to determine whether the providerID is the standard cloud provider ID +// or has been modified by CAS CAPI provider +func TestNodeHasValidProviderID(t *testing.T) { + type testCase struct { + description string + providerID normalizedProviderID + expectedReturn bool + } + + testCases := []testCase{ + { + description: "real looking provider ID should return true", + providerID: normalizedProviderID("fake-provider://a.provider.id-0001"), + expectedReturn: true, + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createDeletingMachineNormalizedProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + { + description: "provider ID created with createPendingDeletionMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createPendingMachineProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + { + description: "provider ID created with createFailedMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := nodeHasValidProviderID(tc.providerID) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %t, observed %t", tc.providerID, tc.expectedReturn, observed) + } + }) + } +} + func Test_machineKeyFromDeletingMachineProviderID(t *testing.T) { type testCase struct { description string @@ -2254,3 +2297,31 @@ func Test_createDeletingMachineNormalizedProviderID(t *testing.T) { }) } } + +// Test_createPendingMachineProviderID tests the creation of a pending machine provider ID +func Test_createPendingMachineProviderID(t *testing.T) { + type testCase struct { + description string + namespace string + name string + expectedReturn string + } + + testCases := []testCase{ + { + description: "namespace and name return proper normalized ID", + namespace: "cluster-api", + name: "id-0001", + expectedReturn: fmt.Sprintf("%s%s_%s", pendingMachinePrefix, "cluster-api", "id-0001"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := createPendingMachineProviderID(tc.namespace, tc.name) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for (namespace %q, name %q), expected %q, observed %q", tc.namespace, tc.name, tc.expectedReturn, observed) + } + }) + } +} diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index d79cda3b7e5..25c0798ba36 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -212,9 +212,7 @@ func (ng *nodegroup) DecreaseTargetSize(delta int) error { // provider. actualNodes := 0 for _, node := range nodes { - if !isPendingMachineProviderID(normalizedProviderID(node.Id)) && - !isFailedMachineProviderID(normalizedProviderID(node.Id)) && - !isDeletingMachineProviderID(normalizedProviderID(node.Id)) { + if nodeHasValidProviderID(normalizedProviderID(node.Id)) { actualNodes += 1 } } From 7b5e10156e348afad2419fae96c490af4124df28 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Wed, 19 Mar 2025 12:30:33 -0700 Subject: [PATCH 21/39] s/nodeHasValidProviderID/isProviderIDNormalized Signed-off-by: Jack Francis --- .../cloudprovider/clusterapi/clusterapi_controller.go | 4 ++-- .../cloudprovider/clusterapi/clusterapi_controller_test.go | 2 +- .../cloudprovider/clusterapi/clusterapi_nodegroup.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index 76ed9051c51..6c95ca23fc0 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -383,10 +383,10 @@ func machineKeyFromFailedProviderID(providerID normalizedProviderID) string { return strings.Replace(namespaceName, "_", "/", 1) } -// nodeHasValidProviderID determines whether a node's providerID is the standard +// isProviderIDNormalized determines whether a node's providerID is the standard // providerID assigned by the cloud provider, or if it has // been modified by the CAS CAPI provider to indicate deleting, pending, or failed -func nodeHasValidProviderID(providerID normalizedProviderID) bool { +func isProviderIDNormalized(providerID normalizedProviderID) bool { return !isDeletingMachineProviderID(providerID) && !isPendingMachineProviderID(providerID) && !isFailedMachineProviderID(providerID) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index a468170bea0..501f89da17d 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -2218,7 +2218,7 @@ func TestNodeHasValidProviderID(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - observed := nodeHasValidProviderID(tc.providerID) + observed := isProviderIDNormalized(tc.providerID) if observed != tc.expectedReturn { t.Fatalf("unexpected return for provider ID %q, expected %t, observed %t", tc.providerID, tc.expectedReturn, observed) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index 25c0798ba36..acfda482fd6 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -212,7 +212,7 @@ func (ng *nodegroup) DecreaseTargetSize(delta int) error { // provider. actualNodes := 0 for _, node := range nodes { - if nodeHasValidProviderID(normalizedProviderID(node.Id)) { + if isProviderIDNormalized(normalizedProviderID(node.Id)) { actualNodes += 1 } } From 5268053d1eae2da87e6209a0c7f1384866332112 Mon Sep 17 00:00:00 2001 From: Yahia Badr <30608188+YahiaBadr@users.noreply.github.com> Date: Thu, 20 Mar 2025 15:04:32 +0100 Subject: [PATCH 22/39] Update default value for scaleDownDelayAfterDelete (#7957) * Update default value for scaleDownDelayAfterDelete Setting defaut value for scaleDownDelayAfterDelete to be scanInterval instead of 0. * Revert the change and fix the flag description --- cluster-autoscaler/config/flags/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-autoscaler/config/flags/flags.go b/cluster-autoscaler/config/flags/flags.go index 0838a8a3652..4d35780d4eb 100644 --- a/cluster-autoscaler/config/flags/flags.go +++ b/cluster-autoscaler/config/flags/flags.go @@ -77,7 +77,7 @@ var ( scaleDownDelayTypeLocal = flag.Bool("scale-down-delay-type-local", false, "Should --scale-down-delay-after-* flags be applied locally per nodegroup or globally across all nodegroups") scaleDownDelayAfterDelete = flag.Duration("scale-down-delay-after-delete", 0, - "How long after node deletion that scale down evaluation resumes, defaults to scanInterval") + "How long after node deletion that scale down evaluation resumes") scaleDownDelayAfterFailure = flag.Duration("scale-down-delay-after-failure", config.DefaultScaleDownDelayAfterFailure, "How long after scale down failure that scale down evaluation resumes") scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", config.DefaultScaleDownUnneededTime, From 455d29039bf6b1eb9f784f498f28769a8698bc21 Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Thu, 20 Mar 2025 14:31:30 -0400 Subject: [PATCH 23/39] Address more comments in AEP --- .../7862-cpu-startup-boost/README.md | 95 ++++++++++++------- 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md index 76689d6182e..8bc236253a8 100644 --- a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md +++ b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md @@ -31,11 +31,12 @@ running in containerized applications, especially Java workloads. This delay can negatively impact the user experience and overall application performance. One potential solution is to provide additional CPU resources to pods during their startup phase, but this can lead to waste if the extra CPU resources are not -set back to their original values after the pods are ready. +set back to their original values after the pods have started up. This proposal allows VPA to boost the CPU request and limit of containers during -the pod startup and to scale the CPU resources back down when the pod is `Ready`, -leveraging the [in-place pod resize Kubernetes feature](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). +the pod startup and to scale the CPU resources back down when the pod is +`Ready` or after certain time has elapsed, leveraging the +[in-place pod resize Kubernetes feature](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). > [!NOTE] > This feature depends on the new `InPlaceOrRecreate` VPA mode: @@ -44,17 +45,16 @@ leveraging the [in-place pod resize Kubernetes feature](https://github.com/kuber ### Goals * Allow VPA to boost the CPU request and limit of a pod's containers during the -pod startup (from creation time until it becomes `Ready`). +pod (re-)creation time. * Allow VPA to scale pods down [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources) to the existing VPA recommendation for that container, if any, or to the CPU resources configured in the pod spec, as soon as their [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) -condition is true. +condition is true and `StartupBoost.CPU.Duration` has elapsed. ### Non-Goals -* Allow VPA to boost CPU resources of pods that are already [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions). -* Allow VPA to boost CPU resources during startup of workloads that have not -configured a [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). +* Allow VPA to boost CPU resources of pods outside of the pod (re-)creation +time. * Allow VPA to boost memory resources. * This is out of scope for now because the in-place pod resize feature [does not support memory limit decrease yet.](https://github.com/kubernetes/enhancements/tree/758ea034908515a934af09d03a927b24186af04c/keps/sig-node/1287-in-place-update-pod-resources#memory-limit-decreases) @@ -69,6 +69,10 @@ boost. with a new `StartupBoostOnly` mode to allow users to only enable the startup boost feature and not vanilla VPA altogether. +* To allow CPU startup boost if a `StartupBoost` config is specified in `Auto` +[`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) +container policies. + ## Design Details ### Workflow @@ -83,9 +87,10 @@ limits to align with its `StartupBoost` policy, if specified, during the pod creation. 1. The VPA Updater monitors pods targeted by the VPA object and when the pod -condition is `Ready`, it scales down the CPU resources to the appropriate -non-boosted value: `existing VPA recommendation for that container` (if any) OR -the `CPU resources configured in the pod spec`. +condition is `Ready` and `StartupBoost.CPU.Duration` has elapsed, it scales +down the CPU resources to the appropriate non-boosted value: +`existing VPA recommendation for that container` (if any) OR the +`CPU resources configured in the pod spec`. * The scale down is applied [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). ### API Changes @@ -96,6 +101,8 @@ and contain the following fields: resource request and limit of the containers' targeted by the VPA object. * `StartupBoost.CPU.Value`: the target value of the CPU request or limit during the startup boost phase. + * [Optional] `StartupBoost.CPU.Duration`: if specified, it indicates for how + long to keep the pod boosted **after** it goes to `Ready`. > [!IMPORTANT] > The boosted CPU value will be capped by @@ -105,6 +112,15 @@ and contain the following fields: > [!IMPORTANT] > Only one of `Factor` or `Value` may be specified per container policy. + +> [!NOTE] +> To ensure that containers are unboosted only after their applications are +> started and ready, it is recommended to configure a +> [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +> for the containers that will be CPU boosted. Check the [Test Plan](#test-plan) +> section for more details on this feature's behavior for different combinations +> of probers + `StartupBoost.CPU.Duration`. + We will also add a new mode to the [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236): * **NEW**: `StartupBoostOnly`: new mode that will allow users to only enable the startup boost feature for a container and not vanilla VPA altogether. @@ -146,21 +162,17 @@ are created/updated: * `StartupBoost.CPU.Value` must be greater than the CPU request or limit of the container during the boost phase, otherwise we risk downscaling the container. -* Workloads must be configured with a Readiness or a Startup probe to be able to -utilize this feature. Therefore, VPA will not boost CPU resources of workloads -that do not configure a Readiness or a Startup probe. - ### Mitigating Failed In-Place Downsizes -The VPA Updater **will not** evict a pod to actuate a startup CPU boost -recommendation if it attempted to apply the recommendation in place and it -failed (see the [scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA +The VPA Updater **will not** evict a pod if it attempted to scaled the pod down +in place (to unboost its CPU resources) and the update failed (see the +[scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA updater will consider that the update failed). This is to avoid an eviction loop: 1. A pod is created and has its CPU resources boosted -1. The pod is ready. VPA Updater tries to downscale the pod in-place and it -fails. +1. The pod meets the conditions to be unboosted. VPA Updater tries to downscale +the pod in-place and it fails. 1. VPA Updater evicts the pod. Logic flow goes back to (1). ### Feature Enablement and Rollback @@ -189,9 +201,9 @@ the following to happen: * admission-controller **to not** boost CPU resources, should it encounter a VPA configured with a `StartupBoost` config and `StartupBoostOnly` or `Auto` `ContainerScalingMode`. - * updater **to not** unboost CPU resources when pods become `Ready`, should it - encounter a VPA configured with a `StartupBoost` config and `StartupBoostOnly` - or `Auto` `ContainerScalingMode`. + * updater **to not** unboost CPU resources when pods meet the scale down + requirements, should it encounter a VPA configured with a `StartupBoost` + config and `StartupBoostOnly` or `Auto` `ContainerScalingMode`. ### Kubernetes Version Compatibility @@ -199,9 +211,8 @@ Similarly to [AEP-4016](https://github.com/kubernetes/autoscaler/tree/master/ver `StartupBoost` configuration and `StartupBoostOnly` mode are built assuming that VPA will be running on a Kubernetes 1.33+ with the beta version of [KEP-1287: In-Place Update of Pod Resources](https://github.com/kubernetes/enhancements/issues/1287) -enabled. If this is not the case, VPA will behave as if the `CPUStartupBoost` -feature gate was disabled (see [Feature Enablement and Rollback](#feature-enablement-and-rollback) -section for details). +enabled. If this is not the case, VPA's attempt to unboost pods may fail and the +pods may remain boosted for their whole lifecycle. ## Test Plan @@ -209,13 +220,22 @@ Other than comprehensive unit tests, we will also add the following scenarios to our e2e tests: * CPU Startup Boost recommendation is applied to pod controlled by VPA until it -becomes `Ready`. Then, the pod is scaled back down in-place. +becomes `Ready` and `StartupBoost.CPU.Duration` has elapsed. Then, the pod is +scaled back down in-place. We'll also test the following sub-cases: * Boost is applied to all containers of a pod. - * Boost is applied to a subset of containers. -* CPU Startup Boost will not be applied if a pod is not configured with a -Readiness or a Startup probe. -* Pod is evicted the first time that an in-place update fails when scaling the -pod back down. And a new CPU boost is not attempted when the pod is recreated. + * Boost is applied only to a subset of containers in a pod. + * Combinations of probes + `StartupBoost.CPU.Duration`: + * No probes and no `StartupBoost.CPU.Duration` specified: unboost will + likely happen immediately. + * No probes and a 60s `StartupBoost.CPU.Duration`: unboost will likely + happen after 60s. + * A readiness/startup probe and no `StartupBoost.CPU.Duration` specified: + unboost will likely as soon as the pod becomes `Ready`. + * A readiness/startup probe and a 60s `StartupBoost.CPU.Duration` + specified: unboost will likely happen 60s **after** the pod becomes `Ready`. + +* Pod is not evicted if the in-place update fails when scaling the pod back +down. ## Examples @@ -224,6 +244,10 @@ scenarios. ### CPU Boost Only +All containers under `example` deployment will receive "regular" VPA updates, +**except for** `boosted-container-name`. `boosted-container-name` will only be +CPU boosted/unboosted, because it has a `StartupBoostOnly` container policy. + ```yaml apiVersion: "autoscaling.k8s.io/v1" kind: VerticalPodAutoscaler @@ -248,6 +272,11 @@ spec: ### CPU Boost and Vanilla VPA +All containers under `example` deployment will receive "regular" VPA updates, +**including** `boosted-container-name`. Additionally, `boosted-container-name` +will be CPU boosted/unboosted, because it has a `StartupBoost` config in its +container policy and `Auto` container policy mode. + ```yaml apiVersion: "autoscaling.k8s.io/v1" kind: VerticalPodAutoscaler @@ -279,5 +308,5 @@ spec: ## Implementation History -* 2025-03-18: Initial version. +* 2025-03-20: Initial version. From 2bbe859154d6978f8bb94247e63b86f4068bda0d Mon Sep 17 00:00:00 2001 From: Omran Date: Thu, 20 Mar 2025 05:34:25 +0000 Subject: [PATCH 24/39] Fix cool down status condition to trigger scale down --- cluster-autoscaler/core/static_autoscaler.go | 32 +++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index 68992602383..6188e1c6690 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -627,6 +627,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr if scaleDownInCooldown { scaleDownStatus.Result = scaledownstatus.ScaleDownInCooldown + a.updateSoftDeletionTaints(allNodes) } else { klog.V(4).Infof("Starting scale down") @@ -645,21 +646,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr a.lastScaleDownDeleteTime = currentTime a.clusterStateRegistry.Recalculate() } - - if scaleDownStatus.Result == scaledownstatus.ScaleDownNoNodeDeleted && - a.AutoscalingContext.AutoscalingOptions.MaxBulkSoftTaintCount != 0 { - taintableNodes := a.scaleDownPlanner.UnneededNodes() - - // Make sure we are only cleaning taints from selected node groups. - selectedNodes := filterNodesFromSelectedGroups(a.CloudProvider, allNodes...) - - // This is a sanity check to make sure `taintableNodes` only includes - // nodes from selected nodes. - taintableNodes = intersectNodes(selectedNodes, taintableNodes) - untaintableNodes := subtractNodes(selectedNodes, taintableNodes) - actuation.UpdateSoftDeletionTaints(a.AutoscalingContext, taintableNodes, untaintableNodes) - } - + a.updateSoftDeletionTaints(allNodes) if typedErr != nil { klog.Errorf("Failed to scale down: %v", typedErr) a.lastScaleDownFailTime = currentTime @@ -679,6 +666,21 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr return nil } +func (a *StaticAutoscaler) updateSoftDeletionTaints(allNodes []*apiv1.Node) { + if a.AutoscalingContext.AutoscalingOptions.MaxBulkSoftTaintCount != 0 { + taintableNodes := a.scaleDownPlanner.UnneededNodes() + + // Make sure we are only cleaning taints from selected node groups. + selectedNodes := filterNodesFromSelectedGroups(a.CloudProvider, allNodes...) + + // This is a sanity check to make sure `taintableNodes` only includes + // nodes from selected nodes. + taintableNodes = intersectNodes(selectedNodes, taintableNodes) + untaintableNodes := subtractNodes(selectedNodes, taintableNodes) + actuation.UpdateSoftDeletionTaints(a.AutoscalingContext, taintableNodes, untaintableNodes) + } +} + func (a *StaticAutoscaler) addUpcomingNodesToClusterSnapshot(upcomingCounts map[string]int, nodeInfosForGroups map[string]*framework.NodeInfo) error { nodeGroups := a.nodeGroupsById() upcomingNodeGroups := make(map[string]int) From a226478f5359c8bdfff72facbb5ed491f6a36990 Mon Sep 17 00:00:00 2001 From: Veer Singh Date: Mon, 24 Mar 2025 04:06:26 +0000 Subject: [PATCH 25/39] pricing changes: updated z3 pricing information --- .../cloudprovider/gce/gce_price_info.go | 800 +++++++++--------- 1 file changed, 416 insertions(+), 384 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/gce/gce_price_info.go b/cluster-autoscaler/cloudprovider/gce/gce_price_info.go index 003d3afde29..f71494d649d 100644 --- a/cluster-autoscaler/cloudprovider/gce/gce_price_info.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_price_info.go @@ -82,7 +82,7 @@ var ( "n2d": 0.027502, "n4": 0.030821, "t2d": 0.027502, - "z3": 0.04965, + "z3": 0.0496531, } predefinedMemoryPricePerHourPerGb = map[string]float64{ "a2": 0.004237, @@ -98,7 +98,7 @@ var ( "n2d": 0.003686, "n4": 0.004131, "t2d": 0.003686, - "z3": 0.00666, + "z3": 0.0066553, } predefinedPreemptibleDiscount = map[string]float64{ "a2": 0.009483 / 0.031611, @@ -114,7 +114,7 @@ var ( "n2d": 0.002773 / 0.027502, "n4": 0.007976 / 0.030821, "t2d": 0.006655 / 0.027502, - "z3": 0.01986 / 0.04965, + "z3": 0.0165 / 0.0496531, } customCpuPricePerHour = map[string]float64{ "e2": 0.022890, @@ -140,389 +140,421 @@ var ( // between the three machine types, the prices for e2-micro and e2-small // are artificially set to be higher than the price of e2-medium. instancePrices = map[string]float64{ - "a2-highgpu-1g": 3.673385, - "a2-highgpu-2g": 7.34677, - "a2-highgpu-4g": 14.69354, - "a2-highgpu-8g": 29.38708, - "a2-megagpu-16g": 55.739504, - "a2-ultragpu-1g": 5.0688, - "a2-ultragpu-2g": 10.1376, - "a2-ultragpu-4g": 20.2752, - "g2-standard-4": 0.76, - "g2-standard-8": 0.91, - "g2-standard-12": 1.06, - "g2-standard-16": 1.20, - "g2-standard-24": 2.11, - "g2-standard-32": 1.79, - "g2-standard-48": 4.23, - "g2-standard-96": 8.46, - "a2-ultragpu-8g": 40.5504, - "c2-standard-4": 0.2088, - "c2-standard-8": 0.4176, - "c2-standard-16": 0.8352, - "c2-standard-30": 1.5660, - "c2-standard-60": 3.1321, - "c2d-highcpu-2": 0.0750, - "c2d-highcpu-4": 0.1499, - "c2d-highcpu-8": 0.2998, - "c2d-highcpu-16": 0.5997, - "c2d-highcpu-32": 1.1994, - "c2d-highcpu-56": 2.0989, - "c2d-highcpu-112": 4.1979, - "c2d-highmem-2": 0.1225, - "c2d-highmem-4": 0.2449, - "c2d-highmem-8": 0.4899, - "c2d-highmem-16": 0.9798, - "c2d-highmem-32": 1.9595, - "c2d-highmem-56": 3.4292, - "c2d-highmem-112": 6.8583, - "c2d-standard-2": 0.0908, - "c2d-standard-4": 0.1816, - "c2d-standard-8": 0.3632, - "c2d-standard-16": 0.7264, - "c2d-standard-32": 1.4528, - "c2d-standard-56": 2.5423, - "c2d-standard-112": 5.0847, - "c3-standard-4": 0.20888, - "c3-standard-8": 0.41776, - "c3-standard-22": 1.14884, - "c3-standard-44": 2.29768, - "c3-standard-88": 4.59536, - "c3-standard-176": 9.19072, - "c3-highmem-4": 0.28184, - "c3-highmem-8": 0.56368, - "c3-highmem-22": 1.55012, - "c3-highmem-44": 3.10024, - "c3-highmem-88": 6.20048, - "c3-highmem-176": 12.40096, - "c3-highcpu-4": 0.1724, - "c3-highcpu-8": 0.3448, - "c3-highcpu-22": 0.9482, - "c3-highcpu-44": 1.8964, - "c3-highcpu-88": 3.7928, - "c3-highcpu-176": 7.5856, - "c3d-standard-4": 0.1816, - "c3d-standard-8": 0.3632, - "c3d-standard-16": 0.7264, - "c3d-standard-30": 1.362, - "c3d-standard-60": 2.724, - "c3d-standard-90": 4.086, - "c3d-standard-180": 8.172, - "c3d-standard-360": 16.344, - "c3d-highmem-4": 0.24496, - "c3d-highmem-8": 0.48992, - "c3d-highmem-16": 0.97984, - "c3d-highmem-30": 1.8372, - "c3d-highmem-60": 3.6744, - "c3d-highmem-90": 5.5116, - "c3d-highmem-180": 11.0232, - "c3d-highmem-360": 22.0464, - "c3d-highcpu-4": 0.14992, - "c3d-highcpu-8": 0.29984, - "c3d-highcpu-16": 0.59968, - "c3d-highcpu-30": 1.1244, - "c3d-highcpu-60": 2.2488, - "c3d-highcpu-90": 3.3732, - "c3d-highcpu-180": 6.7464, - "c3d-highcpu-360": 13.4928, - "e2-highcpu-2": 0.04947, - "e2-highcpu-4": 0.09894, - "e2-highcpu-8": 0.19788, - "e2-highcpu-16": 0.39576, - "e2-highcpu-32": 0.79149, - "e2-highmem-2": 0.09040, - "e2-highmem-4": 0.18080, - "e2-highmem-8": 0.36160, - "e2-highmem-16": 0.72320, - "e2-medium": 0.03351, - "e2-micro": 0.03353, // Should be 0.00838. Set to be > e2-medium. - "e2-small": 0.03352, // Should be 0.01675. Set to be > e2-medium. - "e2-standard-2": 0.06701, - "e2-standard-4": 0.13402, - "e2-standard-8": 0.26805, - "e2-standard-16": 0.53609, - "e2-standard-32": 1.07210, - "f1-micro": 0.0076, - "g1-small": 0.0257, - "m1-megamem-96": 10.6740, - "m1-ultramem-40": 6.3039, - "m1-ultramem-80": 12.6078, - "m1-ultramem-160": 25.2156, - "m2-ultramem-208": 42.186, - "m2-ultramem-416": 84.371, - "m2-megamem-416": 50.372, - "n1-highcpu-2": 0.0709, - "n1-highcpu-4": 0.1418, - "n1-highcpu-8": 0.2836, - "n1-highcpu-16": 0.5672, - "n1-highcpu-32": 1.1344, - "n1-highcpu-64": 2.2688, - "n1-highcpu-96": 3.402, - "n1-highmem-2": 0.1184, - "n1-highmem-4": 0.2368, - "n1-highmem-8": 0.4736, - "n1-highmem-16": 0.9472, - "n1-highmem-32": 1.8944, - "n1-highmem-64": 3.7888, - "n1-highmem-96": 5.6832, - "n1-standard-1": 0.0475, - "n1-standard-2": 0.0950, - "n1-standard-4": 0.1900, - "n1-standard-8": 0.3800, - "n1-standard-16": 0.7600, - "n1-standard-32": 1.5200, - "n1-standard-64": 3.0400, - "n1-standard-96": 4.5600, - "n2-highcpu-2": 0.0717, - "n2-highcpu-4": 0.1434, - "n2-highcpu-8": 0.2868, - "n2-highcpu-16": 0.5736, - "n2-highcpu-32": 1.1471, - "n2-highcpu-48": 1.7207, - "n2-highcpu-64": 2.2943, - "n2-highcpu-80": 2.8678, - "n2-highcpu-96": 3.4414, - "n2-highcpu-128": 4.5886, - "n2-highmem-2": 0.1310, - "n2-highmem-4": 0.2620, - "n2-highmem-8": 0.5241, - "n2-highmem-16": 1.0481, - "n2-highmem-32": 2.0962, - "n2-highmem-48": 3.1443, - "n2-highmem-64": 4.1924, - "n2-highmem-80": 5.2406, - "n2-highmem-96": 6.2886, - "n2-highmem-128": 7.7069, - "n2-standard-2": 0.0971, - "n2-standard-4": 0.1942, - "n2-standard-8": 0.3885, - "n2-standard-16": 0.7769, - "n2-standard-32": 1.5539, - "n2-standard-48": 2.3308, - "n2-standard-64": 3.1078, - "n2-standard-80": 3.8847, - "n2-standard-96": 4.6616, - "n2-standard-128": 6.2156, - "n2d-highcpu-2": 0.0624, - "n2d-highcpu-4": 0.1248, - "n2d-highcpu-8": 0.2495, - "n2d-highcpu-16": 0.4990, - "n2d-highcpu-32": 0.9980, - "n2d-highcpu-48": 1.4970, - "n2d-highcpu-64": 1.9960, - "n2d-highcpu-80": 2.4950, - "n2d-highcpu-96": 2.9940, - "n2d-highcpu-128": 3.9920, - "n2d-highcpu-224": 6.9861, - "n2d-highmem-2": 0.1140, - "n2d-highmem-4": 0.2280, - "n2d-highmem-8": 0.4559, - "n2d-highmem-16": 0.9119, - "n2d-highmem-32": 1.8237, - "n2d-highmem-48": 2.7356, - "n2d-highmem-64": 3.6474, - "n2d-highmem-80": 4.5593, - "n2d-highmem-96": 5.4711, - "n2d-standard-2": 0.0845, - "n2d-standard-4": 0.1690, - "n2d-standard-8": 0.3380, - "n2d-standard-16": 0.6759, - "n2d-standard-32": 1.3519, - "n2d-standard-48": 2.0278, - "n2d-standard-64": 2.7038, - "n2d-standard-80": 3.3797, - "n2d-standard-96": 4.0556, - "n2d-standard-128": 5.4075, - "n2d-standard-224": 9.4632, - "t2d-standard-1": 0.0422, - "t2d-standard-2": 0.0845, - "t2d-standard-4": 0.1690, - "t2d-standard-8": 0.3380, - "t2d-standard-16": 0.6759, - "t2d-standard-32": 1.3519, - "t2d-standard-48": 2.0278, - "t2d-standard-60": 2.5348, - "z3-highmem-88": 13.0, - "z3-highmem-176": 22.05, + "a2-highgpu-1g": 3.673385, + "a2-highgpu-2g": 7.34677, + "a2-highgpu-4g": 14.69354, + "a2-highgpu-8g": 29.38708, + "a2-megagpu-16g": 55.739504, + "a2-ultragpu-1g": 5.0688, + "a2-ultragpu-2g": 10.1376, + "a2-ultragpu-4g": 20.2752, + "g2-standard-4": 0.76, + "g2-standard-8": 0.91, + "g2-standard-12": 1.06, + "g2-standard-16": 1.20, + "g2-standard-24": 2.11, + "g2-standard-32": 1.79, + "g2-standard-48": 4.23, + "g2-standard-96": 8.46, + "a2-ultragpu-8g": 40.5504, + "c2-standard-4": 0.2088, + "c2-standard-8": 0.4176, + "c2-standard-16": 0.8352, + "c2-standard-30": 1.5660, + "c2-standard-60": 3.1321, + "c2d-highcpu-2": 0.0750, + "c2d-highcpu-4": 0.1499, + "c2d-highcpu-8": 0.2998, + "c2d-highcpu-16": 0.5997, + "c2d-highcpu-32": 1.1994, + "c2d-highcpu-56": 2.0989, + "c2d-highcpu-112": 4.1979, + "c2d-highmem-2": 0.1225, + "c2d-highmem-4": 0.2449, + "c2d-highmem-8": 0.4899, + "c2d-highmem-16": 0.9798, + "c2d-highmem-32": 1.9595, + "c2d-highmem-56": 3.4292, + "c2d-highmem-112": 6.8583, + "c2d-standard-2": 0.0908, + "c2d-standard-4": 0.1816, + "c2d-standard-8": 0.3632, + "c2d-standard-16": 0.7264, + "c2d-standard-32": 1.4528, + "c2d-standard-56": 2.5423, + "c2d-standard-112": 5.0847, + "c3-standard-4": 0.20888, + "c3-standard-8": 0.41776, + "c3-standard-22": 1.14884, + "c3-standard-44": 2.29768, + "c3-standard-88": 4.59536, + "c3-standard-176": 9.19072, + "c3-highmem-4": 0.28184, + "c3-highmem-8": 0.56368, + "c3-highmem-22": 1.55012, + "c3-highmem-44": 3.10024, + "c3-highmem-88": 6.20048, + "c3-highmem-176": 12.40096, + "c3-highcpu-4": 0.1724, + "c3-highcpu-8": 0.3448, + "c3-highcpu-22": 0.9482, + "c3-highcpu-44": 1.8964, + "c3-highcpu-88": 3.7928, + "c3-highcpu-176": 7.5856, + "c3d-standard-4": 0.1816, + "c3d-standard-8": 0.3632, + "c3d-standard-16": 0.7264, + "c3d-standard-30": 1.362, + "c3d-standard-60": 2.724, + "c3d-standard-90": 4.086, + "c3d-standard-180": 8.172, + "c3d-standard-360": 16.344, + "c3d-highmem-4": 0.24496, + "c3d-highmem-8": 0.48992, + "c3d-highmem-16": 0.97984, + "c3d-highmem-30": 1.8372, + "c3d-highmem-60": 3.6744, + "c3d-highmem-90": 5.5116, + "c3d-highmem-180": 11.0232, + "c3d-highmem-360": 22.0464, + "c3d-highcpu-4": 0.14992, + "c3d-highcpu-8": 0.29984, + "c3d-highcpu-16": 0.59968, + "c3d-highcpu-30": 1.1244, + "c3d-highcpu-60": 2.2488, + "c3d-highcpu-90": 3.3732, + "c3d-highcpu-180": 6.7464, + "c3d-highcpu-360": 13.4928, + "e2-highcpu-2": 0.04947, + "e2-highcpu-4": 0.09894, + "e2-highcpu-8": 0.19788, + "e2-highcpu-16": 0.39576, + "e2-highcpu-32": 0.79149, + "e2-highmem-2": 0.09040, + "e2-highmem-4": 0.18080, + "e2-highmem-8": 0.36160, + "e2-highmem-16": 0.72320, + "e2-medium": 0.03351, + "e2-micro": 0.03353, // Should be 0.00838. Set to be > e2-medium. + "e2-small": 0.03352, // Should be 0.01675. Set to be > e2-medium. + "e2-standard-2": 0.06701, + "e2-standard-4": 0.13402, + "e2-standard-8": 0.26805, + "e2-standard-16": 0.53609, + "e2-standard-32": 1.07210, + "f1-micro": 0.0076, + "g1-small": 0.0257, + "m1-megamem-96": 10.6740, + "m1-ultramem-40": 6.3039, + "m1-ultramem-80": 12.6078, + "m1-ultramem-160": 25.2156, + "m2-ultramem-208": 42.186, + "m2-ultramem-416": 84.371, + "m2-megamem-416": 50.372, + "n1-highcpu-2": 0.0709, + "n1-highcpu-4": 0.1418, + "n1-highcpu-8": 0.2836, + "n1-highcpu-16": 0.5672, + "n1-highcpu-32": 1.1344, + "n1-highcpu-64": 2.2688, + "n1-highcpu-96": 3.402, + "n1-highmem-2": 0.1184, + "n1-highmem-4": 0.2368, + "n1-highmem-8": 0.4736, + "n1-highmem-16": 0.9472, + "n1-highmem-32": 1.8944, + "n1-highmem-64": 3.7888, + "n1-highmem-96": 5.6832, + "n1-standard-1": 0.0475, + "n1-standard-2": 0.0950, + "n1-standard-4": 0.1900, + "n1-standard-8": 0.3800, + "n1-standard-16": 0.7600, + "n1-standard-32": 1.5200, + "n1-standard-64": 3.0400, + "n1-standard-96": 4.5600, + "n2-highcpu-2": 0.0717, + "n2-highcpu-4": 0.1434, + "n2-highcpu-8": 0.2868, + "n2-highcpu-16": 0.5736, + "n2-highcpu-32": 1.1471, + "n2-highcpu-48": 1.7207, + "n2-highcpu-64": 2.2943, + "n2-highcpu-80": 2.8678, + "n2-highcpu-96": 3.4414, + "n2-highcpu-128": 4.5886, + "n2-highmem-2": 0.1310, + "n2-highmem-4": 0.2620, + "n2-highmem-8": 0.5241, + "n2-highmem-16": 1.0481, + "n2-highmem-32": 2.0962, + "n2-highmem-48": 3.1443, + "n2-highmem-64": 4.1924, + "n2-highmem-80": 5.2406, + "n2-highmem-96": 6.2886, + "n2-highmem-128": 7.7069, + "n2-standard-2": 0.0971, + "n2-standard-4": 0.1942, + "n2-standard-8": 0.3885, + "n2-standard-16": 0.7769, + "n2-standard-32": 1.5539, + "n2-standard-48": 2.3308, + "n2-standard-64": 3.1078, + "n2-standard-80": 3.8847, + "n2-standard-96": 4.6616, + "n2-standard-128": 6.2156, + "n2d-highcpu-2": 0.0624, + "n2d-highcpu-4": 0.1248, + "n2d-highcpu-8": 0.2495, + "n2d-highcpu-16": 0.4990, + "n2d-highcpu-32": 0.9980, + "n2d-highcpu-48": 1.4970, + "n2d-highcpu-64": 1.9960, + "n2d-highcpu-80": 2.4950, + "n2d-highcpu-96": 2.9940, + "n2d-highcpu-128": 3.9920, + "n2d-highcpu-224": 6.9861, + "n2d-highmem-2": 0.1140, + "n2d-highmem-4": 0.2280, + "n2d-highmem-8": 0.4559, + "n2d-highmem-16": 0.9119, + "n2d-highmem-32": 1.8237, + "n2d-highmem-48": 2.7356, + "n2d-highmem-64": 3.6474, + "n2d-highmem-80": 4.5593, + "n2d-highmem-96": 5.4711, + "n2d-standard-2": 0.0845, + "n2d-standard-4": 0.1690, + "n2d-standard-8": 0.3380, + "n2d-standard-16": 0.6759, + "n2d-standard-32": 1.3519, + "n2d-standard-48": 2.0278, + "n2d-standard-64": 2.7038, + "n2d-standard-80": 3.3797, + "n2d-standard-96": 4.0556, + "n2d-standard-128": 5.4075, + "n2d-standard-224": 9.4632, + "t2d-standard-1": 0.0422, + "t2d-standard-2": 0.0845, + "t2d-standard-4": 0.1690, + "t2d-standard-8": 0.3380, + "t2d-standard-16": 0.6759, + "t2d-standard-32": 1.3519, + "t2d-standard-48": 2.0278, + "t2d-standard-60": 2.5348, + "z3-highmem-176": 21.980576, + "z3-highmem-8": 1.145745, + "z3-highmem-14": 2.085698, + "z3-highmem-22": 3.231443, + "z3-highmem-30": 4.377188, + "z3-highmem-36": 5.317141, + "z3-highmem-44": 6.462886, + "z3-highmem-8-highlssd": 1.145745, + "z3-highmem-16-highlssd": 2.291489, + "z3-highmem-22-highlssd": 3.231443, + "z3-highmem-32-highlssd": 4.582979, + "z3-highmem-44-highlssd": 6.462886, + "z3-highmem-88-highlssd": 12.925772, + "z3-highmem-14-standardlssd": 1.763118, + "z3-highmem-22-standardlssd": 2.908862, + "z3-highmem-44-standardlssd": 5.495144, + "z3-highmem-88-standardlssd": 10.990288, + "z3-highmem-176-standardlssd": 21.980576, } preemptiblePrices = map[string]float64{ - "a2-highgpu-1g": 1.102016, - "a2-highgpu-2g": 2.204031, - "a2-highgpu-4g": 4.408062, - "a2-highgpu-8g": 8.816124, - "a2-megagpu-16g": 16.721851, - "a2-ultragpu-1g": 1.6, - "a2-ultragpu-2g": 3.2, - "a2-ultragpu-4g": 6.4, - "a2-ultragpu-8g": 12.8, - "g2-standard-4": 0.23, - "g2-standard-8": 0.27, - "g2-standard-12": 0.32, - "g2-standard-16": 0.36, - "g2-standard-24": 0.63, - "g2-standard-32": 0.54, - "g2-standard-48": 1.27, - "g2-standard-96": 2.54, - "c2-standard-4": 0.0505, - "c2-standard-8": 0.1011, - "c2-standard-16": 0.2021, - "c2-standard-30": 0.3790, - "c2-standard-60": 0.7579, - "c2d-highcpu-2": 0.0181, - "c2d-highcpu-4": 0.0363, - "c2d-highcpu-8": 0.0726, - "c2d-highcpu-16": 0.1451, - "c2d-highcpu-32": 0.2902, - "c2d-highcpu-56": 0.5079, - "c2d-highcpu-112": 1.0158, - "c2d-highmem-2": 0.0296, - "c2d-highmem-4": 0.0593, - "c2d-highmem-8": 0.1185, - "c2d-highmem-16": 0.2371, - "c2d-highmem-32": 0.4742, - "c2d-highmem-56": 0.8298, - "c2d-highmem-112": 1.6596, - "c2d-standard-2": 0.0220, - "c2d-standard-4": 0.0439, - "c2d-standard-8": 0.0879, - "c2d-standard-16": 0.1758, - "c2d-standard-32": 0.3516, - "c2d-standard-56": 0.6152, - "c2d-standard-112": 1.2304, - "c3-standard-4": 0.018952, - "c3-standard-8": 0.037904, - "c3-standard-22": 0.104236, - "c3-standard-44": 0.208472, - "c3-standard-88": 0.416944, - "c3-standard-176": 0.833888, - "c3-highmem-4": 0.02556, - "c3-highmem-8": 0.05112, - "c3-highmem-22": 0.14058, - "c3-highmem-44": 0.28116, - "c3-highmem-88": 0.56232, - "c3-highmem-176": 1.12464, - "c3-highcpu-4": 0.015648, - "c3-highcpu-8": 0.031296, - "c3-highcpu-22": 0.086064, - "c3-highcpu-44": 0.172128, - "c3-highcpu-88": 0.344256, - "c3-highcpu-176": 0.688512, - "e2-highcpu-2": 0.01484, - "e2-highcpu-4": 0.02968, - "e2-highcpu-8": 0.05936, - "e2-highcpu-16": 0.11873, - "e2-highcpu-32": 0.23744, - "e2-highmem-2": 0.02712, - "e2-highmem-4": 0.05424, - "e2-highmem-8": 0.10848, - "e2-highmem-16": 0.21696, - "e2-medium": 0.01005, - "e2-micro": 0.01007, // Should be 0.00251. Set to be > e2-medium. - "e2-small": 0.01006, // Should be 0.00503. Set to be > e2-medium. - "e2-standard-2": 0.02010, - "e2-standard-4": 0.04021, - "e2-standard-8": 0.08041, - "e2-standard-16": 0.16083, - "e2-standard-32": 0.32163, - "f1-micro": 0.0035, - "g1-small": 0.0070, - "m1-megamem-96": 2.2600, - "m1-ultramem-40": 1.3311, - "m1-ultramem-80": 2.6622, - "m1-ultramem-160": 5.3244, - "n1-highcpu-2": 0.0150, - "n1-highcpu-4": 0.0300, - "n1-highcpu-8": 0.0600, - "n1-highcpu-16": 0.1200, - "n1-highcpu-32": 0.2400, - "n1-highcpu-64": 0.4800, - "n1-highcpu-96": 0.7200, - "n1-highmem-2": 0.0250, - "n1-highmem-4": 0.0500, - "n1-highmem-8": 0.1000, - "n1-highmem-16": 0.2000, - "n1-highmem-32": 0.4000, - "n1-highmem-64": 0.8000, - "n1-highmem-96": 1.2000, - "n1-standard-1": 0.0100, - "n1-standard-2": 0.0200, - "n1-standard-4": 0.0400, - "n1-standard-8": 0.0800, - "n1-standard-16": 0.1600, - "n1-standard-32": 0.3200, - "n1-standard-64": 0.6400, - "n1-standard-96": 0.9600, - "n2-highcpu-2": 0.0173, - "n2-highcpu-4": 0.0347, - "n2-highcpu-8": 0.0694, - "n2-highcpu-16": 0.1388, - "n2-highcpu-32": 0.2776, - "n2-highcpu-48": 0.4164, - "n2-highcpu-64": 0.5552, - "n2-highcpu-80": 0.6940, - "n2-highcpu-96": 0.8328, - "n2-highcpu-128": 1.1104, - "n2-highmem-2": 0.0317, - "n2-highmem-4": 0.0634, - "n2-highmem-8": 0.1268, - "n2-highmem-16": 0.2536, - "n2-highmem-32": 0.5073, - "n2-highmem-48": 0.7609, - "n2-highmem-64": 1.0145, - "n2-highmem-80": 1.2681, - "n2-highmem-96": 1.5218, - "n2-highmem-128": 1.8691, - "n2-standard-2": 0.0235, - "n2-standard-4": 0.0470, - "n2-standard-8": 0.0940, - "n2-standard-16": 0.1880, - "n2-standard-32": 0.3760, - "n2-standard-48": 0.5640, - "n2-standard-64": 0.7520, - "n2-standard-80": 0.9400, - "n2-standard-96": 1.128, - "n2-standard-128": 1.504, - "n2d-highcpu-2": 0.00629, - "n2d-highcpu-4": 0.01258, - "n2d-highcpu-8": 0.02516, - "n2d-highcpu-16": 0.05032, - "n2d-highcpu-32": 0.10064, - "n2d-highcpu-48": 0.15096, - "n2d-highcpu-64": 0.20128, - "n2d-highcpu-80": 0.2516, - "n2d-highcpu-96": 0.30192, - "n2d-highcpu-128": 0.40256, - "n2d-highcpu-224": 0.70448, - "n2d-highmem-2": 0.011498, - "n2d-highmem-4": 0.022996, - "n2d-highmem-8": 0.045992, - "n2d-highmem-16": 0.091984, - "n2d-highmem-32": 0.183968, - "n2d-highmem-48": 0.275952, - "n2d-highmem-64": 0.367936, - "n2d-highmem-80": 0.45992, - "n2d-highmem-96": 0.551904, - "n2d-standard-2": 0.008522, - "n2d-standard-4": 0.017044, - "n2d-standard-8": 0.034088, - "n2d-standard-16": 0.068176, - "n2d-standard-32": 0.136352, - "n2d-standard-48": 0.204528, - "n2d-standard-64": 0.272704, - "n2d-standard-80": 0.34088, - "n2d-standard-96": 0.409056, - "n2d-standard-128": 0.545408, - "n2d-standard-224": 0.954464, - "t2d-standard-1": 0.0102, - "t2d-standard-2": 0.0204, - "t2d-standard-4": 0.0409, - "t2d-standard-8": 0.0818, - "t2d-standard-16": 0.1636, - "t2d-standard-32": 0.3271, - "t2d-standard-48": 0.4907, - "t2d-standard-60": 0.6134, - "z3-highmem-88": 5.2, - "z3-highmem-176": 8.82, + "a2-highgpu-1g": 1.102016, + "a2-highgpu-2g": 2.204031, + "a2-highgpu-4g": 4.408062, + "a2-highgpu-8g": 8.816124, + "a2-megagpu-16g": 16.721851, + "a2-ultragpu-1g": 1.6, + "a2-ultragpu-2g": 3.2, + "a2-ultragpu-4g": 6.4, + "a2-ultragpu-8g": 12.8, + "g2-standard-4": 0.23, + "g2-standard-8": 0.27, + "g2-standard-12": 0.32, + "g2-standard-16": 0.36, + "g2-standard-24": 0.63, + "g2-standard-32": 0.54, + "g2-standard-48": 1.27, + "g2-standard-96": 2.54, + "c2-standard-4": 0.0505, + "c2-standard-8": 0.1011, + "c2-standard-16": 0.2021, + "c2-standard-30": 0.3790, + "c2-standard-60": 0.7579, + "c2d-highcpu-2": 0.0181, + "c2d-highcpu-4": 0.0363, + "c2d-highcpu-8": 0.0726, + "c2d-highcpu-16": 0.1451, + "c2d-highcpu-32": 0.2902, + "c2d-highcpu-56": 0.5079, + "c2d-highcpu-112": 1.0158, + "c2d-highmem-2": 0.0296, + "c2d-highmem-4": 0.0593, + "c2d-highmem-8": 0.1185, + "c2d-highmem-16": 0.2371, + "c2d-highmem-32": 0.4742, + "c2d-highmem-56": 0.8298, + "c2d-highmem-112": 1.6596, + "c2d-standard-2": 0.0220, + "c2d-standard-4": 0.0439, + "c2d-standard-8": 0.0879, + "c2d-standard-16": 0.1758, + "c2d-standard-32": 0.3516, + "c2d-standard-56": 0.6152, + "c2d-standard-112": 1.2304, + "c3-standard-4": 0.018952, + "c3-standard-8": 0.037904, + "c3-standard-22": 0.104236, + "c3-standard-44": 0.208472, + "c3-standard-88": 0.416944, + "c3-standard-176": 0.833888, + "c3-highmem-4": 0.02556, + "c3-highmem-8": 0.05112, + "c3-highmem-22": 0.14058, + "c3-highmem-44": 0.28116, + "c3-highmem-88": 0.56232, + "c3-highmem-176": 1.12464, + "c3-highcpu-4": 0.015648, + "c3-highcpu-8": 0.031296, + "c3-highcpu-22": 0.086064, + "c3-highcpu-44": 0.172128, + "c3-highcpu-88": 0.344256, + "c3-highcpu-176": 0.688512, + "e2-highcpu-2": 0.01484, + "e2-highcpu-4": 0.02968, + "e2-highcpu-8": 0.05936, + "e2-highcpu-16": 0.11873, + "e2-highcpu-32": 0.23744, + "e2-highmem-2": 0.02712, + "e2-highmem-4": 0.05424, + "e2-highmem-8": 0.10848, + "e2-highmem-16": 0.21696, + "e2-medium": 0.01005, + "e2-micro": 0.01007, // Should be 0.00251. Set to be > e2-medium. + "e2-small": 0.01006, // Should be 0.00503. Set to be > e2-medium. + "e2-standard-2": 0.02010, + "e2-standard-4": 0.04021, + "e2-standard-8": 0.08041, + "e2-standard-16": 0.16083, + "e2-standard-32": 0.32163, + "f1-micro": 0.0035, + "g1-small": 0.0070, + "m1-megamem-96": 2.2600, + "m1-ultramem-40": 1.3311, + "m1-ultramem-80": 2.6622, + "m1-ultramem-160": 5.3244, + "n1-highcpu-2": 0.0150, + "n1-highcpu-4": 0.0300, + "n1-highcpu-8": 0.0600, + "n1-highcpu-16": 0.1200, + "n1-highcpu-32": 0.2400, + "n1-highcpu-64": 0.4800, + "n1-highcpu-96": 0.7200, + "n1-highmem-2": 0.0250, + "n1-highmem-4": 0.0500, + "n1-highmem-8": 0.1000, + "n1-highmem-16": 0.2000, + "n1-highmem-32": 0.4000, + "n1-highmem-64": 0.8000, + "n1-highmem-96": 1.2000, + "n1-standard-1": 0.0100, + "n1-standard-2": 0.0200, + "n1-standard-4": 0.0400, + "n1-standard-8": 0.0800, + "n1-standard-16": 0.1600, + "n1-standard-32": 0.3200, + "n1-standard-64": 0.6400, + "n1-standard-96": 0.9600, + "n2-highcpu-2": 0.0173, + "n2-highcpu-4": 0.0347, + "n2-highcpu-8": 0.0694, + "n2-highcpu-16": 0.1388, + "n2-highcpu-32": 0.2776, + "n2-highcpu-48": 0.4164, + "n2-highcpu-64": 0.5552, + "n2-highcpu-80": 0.6940, + "n2-highcpu-96": 0.8328, + "n2-highcpu-128": 1.1104, + "n2-highmem-2": 0.0317, + "n2-highmem-4": 0.0634, + "n2-highmem-8": 0.1268, + "n2-highmem-16": 0.2536, + "n2-highmem-32": 0.5073, + "n2-highmem-48": 0.7609, + "n2-highmem-64": 1.0145, + "n2-highmem-80": 1.2681, + "n2-highmem-96": 1.5218, + "n2-highmem-128": 1.8691, + "n2-standard-2": 0.0235, + "n2-standard-4": 0.0470, + "n2-standard-8": 0.0940, + "n2-standard-16": 0.1880, + "n2-standard-32": 0.3760, + "n2-standard-48": 0.5640, + "n2-standard-64": 0.7520, + "n2-standard-80": 0.9400, + "n2-standard-96": 1.128, + "n2-standard-128": 1.504, + "n2d-highcpu-2": 0.00629, + "n2d-highcpu-4": 0.01258, + "n2d-highcpu-8": 0.02516, + "n2d-highcpu-16": 0.05032, + "n2d-highcpu-32": 0.10064, + "n2d-highcpu-48": 0.15096, + "n2d-highcpu-64": 0.20128, + "n2d-highcpu-80": 0.2516, + "n2d-highcpu-96": 0.30192, + "n2d-highcpu-128": 0.40256, + "n2d-highcpu-224": 0.70448, + "n2d-highmem-2": 0.011498, + "n2d-highmem-4": 0.022996, + "n2d-highmem-8": 0.045992, + "n2d-highmem-16": 0.091984, + "n2d-highmem-32": 0.183968, + "n2d-highmem-48": 0.275952, + "n2d-highmem-64": 0.367936, + "n2d-highmem-80": 0.45992, + "n2d-highmem-96": 0.551904, + "n2d-standard-2": 0.008522, + "n2d-standard-4": 0.017044, + "n2d-standard-8": 0.034088, + "n2d-standard-16": 0.068176, + "n2d-standard-32": 0.136352, + "n2d-standard-48": 0.204528, + "n2d-standard-64": 0.272704, + "n2d-standard-80": 0.34088, + "n2d-standard-96": 0.409056, + "n2d-standard-128": 0.545408, + "n2d-standard-224": 0.954464, + "t2d-standard-1": 0.0102, + "t2d-standard-2": 0.0204, + "t2d-standard-4": 0.0409, + "t2d-standard-8": 0.0818, + "t2d-standard-16": 0.1636, + "t2d-standard-32": 0.3271, + "t2d-standard-48": 0.4907, + "t2d-standard-60": 0.6134, + "z3-highmem-176": 7.568291, + "z3-highmem-8": 0.402664, + "z3-highmem-14": 0.736921, + "z3-highmem-22": 1.139585, + "z3-highmem-30": 1.542249, + "z3-highmem-36": 1.876505, + "z3-highmem-44": 2.279170, + "z3-highmem-8-highlssd": 0.402664, + "z3-highmem-16-highlssd": 0.805329, + "z3-highmem-22-highlssd": 1.139585, + "z3-highmem-32-highlssd": 1.610657, + "z3-highmem-44-highlssd": 2.279170, + "z3-highmem-88-highlssd": 4.558339, + "z3-highmem-14-standardlssd": 0.607888, + "z3-highmem-22-standardlssd": 1.010553, + "z3-highmem-44-standardlssd": 1.892073, + "z3-highmem-88-standardlssd": 3.784146, + "z3-highmem-176-standardlssd": 7.568291, } gpuPrices = map[string]float64{ "nvidia-tesla-t4": 0.35, From 696af986ed69935c7eeba8646128227f95a3602e Mon Sep 17 00:00:00 2001 From: Omran Date: Wed, 12 Mar 2025 21:58:30 +0000 Subject: [PATCH 26/39] Add time based drainability rule for non-pdb-assigned system pods --- .../config/autoscaling_options.go | 2 + cluster-autoscaler/config/flags/flags.go | 4 +- cluster-autoscaler/simulator/drain_test.go | 45 ++++++++++++++++++- .../simulator/drainability/rules/rules.go | 2 +- .../drainability/rules/system/rule.go | 26 +++++++++-- .../drainability/rules/system/rule_test.go | 38 +++++++++++++++- .../simulator/options/nodedelete.go | 6 +++ 7 files changed, 114 insertions(+), 9 deletions(-) diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index d9d139763a0..cedc37e71ff 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -276,6 +276,8 @@ type AutoscalingOptions struct { // MinReplicaCount controls the minimum number of replicas that a replica set or replication controller should have // to allow their pods deletion in scale down MinReplicaCount int + // BspDisruptionTimeout is the timeout after which CA will evict non-pdb-assigned blocking system pods + BspDisruptionTimeout time.Duration // NodeDeleteDelayAfterTaint is the duration to wait before deleting a node after tainting it NodeDeleteDelayAfterTaint time.Duration // NodeGroupSetRatio is a collection of ratios used by CA used to make scaling decisions. diff --git a/cluster-autoscaler/config/flags/flags.go b/cluster-autoscaler/config/flags/flags.go index 4d35780d4eb..11533a08221 100644 --- a/cluster-autoscaler/config/flags/flags.go +++ b/cluster-autoscaler/config/flags/flags.go @@ -193,10 +193,11 @@ var ( recordDuplicatedEvents = flag.Bool("record-duplicated-events", false, "enable duplication of similar events within a 5 minute window.") maxNodesPerScaleUp = flag.Int("max-nodes-per-scaleup", 1000, "Max nodes added in a single scale-up. This is intended strictly for optimizing CA algorithm latency and not a tool to rate-limit scale-up throughput.") maxNodeGroupBinpackingDuration = flag.Duration("max-nodegroup-binpacking-duration", 10*time.Second, "Maximum time that will be spent in binpacking simulation for each NodeGroup.") - skipNodesWithSystemPods = flag.Bool("skip-nodes-with-system-pods", true, "If true cluster autoscaler will never delete nodes with pods from kube-system (except for DaemonSet or mirror pods)") + skipNodesWithSystemPods = flag.Bool("skip-nodes-with-system-pods", true, "If true cluster autoscaler will wait for --blocking-system-pod-distruption-timeout before deleting nodes with pods from kube-system (except for DaemonSet or mirror pods)") skipNodesWithLocalStorage = flag.Bool("skip-nodes-with-local-storage", true, "If true cluster autoscaler will never delete nodes with pods with local storage, e.g. EmptyDir or HostPath") skipNodesWithCustomControllerPods = flag.Bool("skip-nodes-with-custom-controller-pods", true, "If true cluster autoscaler will never delete nodes with pods owned by custom controllers") minReplicaCount = flag.Int("min-replica-count", 0, "Minimum number or replicas that a replica set or replication controller should have to allow their pods deletion in scale down") + bspDisruptionTimeout = flag.Duration("blocking-system-pod-distruption-timeout", time.Hour, "The timeout after which CA will evict non-pdb-assigned blocking system pods, applicable only when --skip-nodes-with-system-pods is set to true") nodeDeleteDelayAfterTaint = flag.Duration("node-delete-delay-after-taint", 5*time.Second, "How long to wait before deleting a node after tainting it") scaleDownSimulationTimeout = flag.Duration("scale-down-simulation-timeout", 30*time.Second, "How long should we run scale down simulation.") maxCapacityMemoryDifferenceRatio = flag.Float64("memory-difference-ratio", config.DefaultMaxCapacityMemoryDifferenceRatio, "Maximum difference in memory capacity between two similar node groups to be considered for balancing. Value is a ratio of the smaller node group's memory capacity.") @@ -370,6 +371,7 @@ func createAutoscalingOptions() config.AutoscalingOptions { SkipNodesWithSystemPods: *skipNodesWithSystemPods, SkipNodesWithLocalStorage: *skipNodesWithLocalStorage, MinReplicaCount: *minReplicaCount, + BspDisruptionTimeout: *bspDisruptionTimeout, NodeDeleteDelayAfterTaint: *nodeDeleteDelayAfterTaint, ScaleDownSimulationTimeout: *scaleDownSimulationTimeout, SkipNodesWithCustomControllerPods: *skipNodesWithCustomControllerPods, diff --git a/cluster-autoscaler/simulator/drain_test.go b/cluster-autoscaler/simulator/drain_test.go index bb7de730764..3b2faa84539 100644 --- a/cluster-autoscaler/simulator/drain_test.go +++ b/cluster-autoscaler/simulator/drain_test.go @@ -43,7 +43,11 @@ import ( func TestGetPodsToMove(t *testing.T) { var ( - testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + bspDisruptionTimeout = time.Hour + creationTimeBeforeBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(-time.Minute) + creationTimeAfterBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(time.Minute) + replicas = int32(5) unreplicatedPod = &apiv1.Pod{ @@ -68,6 +72,22 @@ func TestGetPodsToMove(t *testing.T) { OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), }, } + drainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeBeforeBspDisturptionTimeout}, + }, + } + nonDrainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeAfterBspDisturptionTimeout}, + }, + } localStoragePod = &apiv1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "localStoragePod", @@ -541,6 +561,28 @@ func TestGetPodsToMove(t *testing.T) { Reason: drain.UnmovableKubeSystemPod, }, }, + { + desc: "Kube-system no pdb system pods blocking", + pods: []*apiv1.Pod{nonDrainableBlockingSystemPod}, + wantErr: true, + wantBlocking: &drain.BlockingPod{ + Pod: nonDrainableBlockingSystemPod, + Reason: drain.UnmovableKubeSystemPod, + }}, + { + desc: "Kube-system no pdb system pods allowing", + pods: []*apiv1.Pod{drainableBlockingSystemPod}, + wantPods: []*apiv1.Pod{drainableBlockingSystemPod}, + }, + { + desc: "Kube-system no pdb system pods blocking", + pods: []*apiv1.Pod{drainableBlockingSystemPod, nonDrainableBlockingSystemPod}, + wantErr: true, + wantBlocking: &drain.BlockingPod{ + Pod: nonDrainableBlockingSystemPod, + Reason: drain.UnmovableKubeSystemPod, + }, + }, { desc: "Local storage", pods: []*apiv1.Pod{localStoragePod}, @@ -771,6 +813,7 @@ func TestGetPodsToMove(t *testing.T) { SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, SkipNodesWithCustomControllerPods: true, + BspDisruptionTimeout: bspDisruptionTimeout, } rules := append(tc.rules, rules.Default(deleteOptions)...) tracker := pdb.NewBasicRemainingPdbTracker() diff --git a/cluster-autoscaler/simulator/drainability/rules/rules.go b/cluster-autoscaler/simulator/drainability/rules/rules.go index be8d00cb415..e5103791299 100644 --- a/cluster-autoscaler/simulator/drainability/rules/rules.go +++ b/cluster-autoscaler/simulator/drainability/rules/rules.go @@ -65,7 +65,7 @@ func Default(deleteOptions options.NodeDeleteOptions) Rules { // Blocking checks {rule: replicated.New(deleteOptions.SkipNodesWithCustomControllerPods)}, - {rule: system.New(), skip: !deleteOptions.SkipNodesWithSystemPods}, + {rule: system.New(deleteOptions.BspDisruptionTimeout), skip: !deleteOptions.SkipNodesWithSystemPods}, {rule: notsafetoevict.New()}, {rule: localstorage.New(), skip: !deleteOptions.SkipNodesWithLocalStorage}, {rule: pdbrule.New()}, diff --git a/cluster-autoscaler/simulator/drainability/rules/system/rule.go b/cluster-autoscaler/simulator/drainability/rules/system/rule.go index fae89fb128a..adf2eca2fed 100644 --- a/cluster-autoscaler/simulator/drainability/rules/system/rule.go +++ b/cluster-autoscaler/simulator/drainability/rules/system/rule.go @@ -18,6 +18,7 @@ package system import ( "fmt" + "time" apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability" @@ -25,12 +26,17 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/drain" ) +// KubeSystemNamespace is the namespase includes system pods +const KubeSystemNamespace = "kube-system" + // Rule is a drainability rule on how to handle system pods. -type Rule struct{} +type Rule struct { + BspDisruptionTimeout time.Duration +} // New creates a new Rule. -func New() *Rule { - return &Rule{} +func New(bspDisruptionTimeout time.Duration) *Rule { + return &Rule{BspDisruptionTimeout: bspDisruptionTimeout} } // Name returns the name of the rule. @@ -40,8 +46,20 @@ func (r *Rule) Name() string { // Drainable decides what to do with system pods on node drain. func (r *Rule) Drainable(drainCtx *drainability.DrainContext, pod *apiv1.Pod, _ *framework.NodeInfo) drainability.Status { - if pod.Namespace == "kube-system" && len(drainCtx.RemainingPdbTracker.MatchingPdbs(pod)) == 0 { + if isBlockingSystemPod(drainCtx, pod) { + if r.isBspPassedDisruptionTimeout(pod, drainCtx.Timestamp) { + return drainability.NewDrainableStatus() + } return drainability.NewBlockedStatus(drain.UnmovableKubeSystemPod, fmt.Errorf("non-daemonset, non-mirrored, non-pdb-assigned kube-system pod present: %s", pod.Name)) } return drainability.NewUndefinedStatus() } + +func isBlockingSystemPod(drainCtx *drainability.DrainContext, pod *apiv1.Pod) bool { + return pod.Namespace == KubeSystemNamespace && len(drainCtx.RemainingPdbTracker.MatchingPdbs(pod)) == 0 +} + +func (r *Rule) isBspPassedDisruptionTimeout(pod *apiv1.Pod, drainTime time.Time) bool { + return !pod.ObjectMeta.CreationTimestamp.IsZero() && + drainTime.After(pod.ObjectMeta.CreationTimestamp.Add(r.BspDisruptionTimeout)) +} diff --git a/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go b/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go index 9c34b808220..0dfcfb7888a 100644 --- a/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go +++ b/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go @@ -34,7 +34,11 @@ import ( func TestDrainable(t *testing.T) { var ( - testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + bspDisruptionTimeout = time.Minute + creationTimeBeforeBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(-time.Second) + creationTimeAfterBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(time.Second) + replicas = int32(5) rc = apiv1.ReplicationController{ @@ -84,6 +88,24 @@ func TestDrainable(t *testing.T) { }, } + drainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: test.GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeBeforeBspDisturptionTimeout}, + }, + } + + nonDrainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: test.GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeAfterBspDisturptionTimeout}, + }, + } + emptyPDB = &policyv1.PodDisruptionBudget{} kubeSystemPDB = &policyv1.PodDisruptionBudget{ @@ -164,6 +186,18 @@ func TestDrainable(t *testing.T) { wantReason: drain.UnmovableKubeSystemPod, wantError: true, }, + "block non-pdb system pod existing for less than BspDisruptionTimeout": { + pod: nonDrainableBlockingSystemPod, + rcs: []*apiv1.ReplicationController{&kubeSystemRc}, + pdbs: []*policyv1.PodDisruptionBudget{emptyPDB}, + wantReason: drain.UnmovableKubeSystemPod, + wantError: true, + }, + "allow non-pdb system pod existing for more than BspDisruptionTimeout": { + pod: drainableBlockingSystemPod, + rcs: []*apiv1.ReplicationController{&kubeSystemRc}, + pdbs: []*policyv1.PodDisruptionBudget{kubeSystemPDB}, + }, } { t.Run(desc, func(t *testing.T) { tracker := pdb.NewBasicRemainingPdbTracker() @@ -173,7 +207,7 @@ func TestDrainable(t *testing.T) { RemainingPdbTracker: tracker, Timestamp: testTime, } - status := New().Drainable(drainCtx, test.pod, nil) + status := New(bspDisruptionTimeout).Drainable(drainCtx, test.pod, nil) assert.Equal(t, test.wantReason, status.BlockingReason) assert.Equal(t, test.wantError, status.Error != nil) }) diff --git a/cluster-autoscaler/simulator/options/nodedelete.go b/cluster-autoscaler/simulator/options/nodedelete.go index 6b6e17a1b7e..e56eb74f2f9 100644 --- a/cluster-autoscaler/simulator/options/nodedelete.go +++ b/cluster-autoscaler/simulator/options/nodedelete.go @@ -17,6 +17,8 @@ limitations under the License. package options import ( + "time" + "k8s.io/autoscaler/cluster-autoscaler/config" ) @@ -35,6 +37,9 @@ type NodeDeleteOptions struct { // set or replication controller should have to allow pod deletion during // scale down. MinReplicaCount int + // BspDisruptionTimeout is the timeout after which CA will evict + // non-pdb-assigned blocking system pods + BspDisruptionTimeout time.Duration } // NewNodeDeleteOptions returns new node delete options extracted from autoscaling options. @@ -44,5 +49,6 @@ func NewNodeDeleteOptions(opts config.AutoscalingOptions) NodeDeleteOptions { SkipNodesWithLocalStorage: opts.SkipNodesWithLocalStorage, SkipNodesWithCustomControllerPods: opts.SkipNodesWithCustomControllerPods, MinReplicaCount: opts.MinReplicaCount, + BspDisruptionTimeout: opts.BspDisruptionTimeout, } } From 72c2f93c7c3f880a17a7fcdd9604e5fccc759d0b Mon Sep 17 00:00:00 2001 From: Plamen Kokanov Date: Mon, 24 Mar 2025 16:24:16 +0200 Subject: [PATCH 27/39] Describe why the additional bucket handling is necessary --- .../pkg/recommender/util/histogram.go | 8 ++++++ .../pkg/recommender/util/histogram_test.go | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go index 08eeca55c5f..2dd078a498a 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go @@ -277,6 +277,14 @@ func (h *histogram) LoadFromCheckpoint(checkpoint *vpa_types.HistogramCheckpoint h.bucketWeight[bucket] += float64(weight) * ratio } h.totalWeight += checkpoint.TotalWeight + + // In some cases where the weight of the max bucket is close (equal or less) to `MaxCheckpointWeight` times epsilon + // and there are buckets with weights slightly higher or equal to epsilon, saving the histogram to a checkpoint and + // then loading it will cause the weights that are close to epsilon to become smaller than epsilon due to rounding errors + // and differences between the load and save algorithm. If one of those weights is the min weight, this will cause the + // histogram to incorrectly become "empty" and the `Percentile(...)` function to always return 0. + // To cover for such cases, the min and max buckets are updated here, so that those less than epsilon are dropped. + // For more information check https://github.com/kubernetes/autoscaler/issues/7726 h.updateMinAndMaxBucket() return nil diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go index 259c0646389..8691c0557ee 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go @@ -266,6 +266,33 @@ func TestHistogramLoadFromCheckpointReturnsErrorOnNilInput(t *testing.T) { } func TestHistogramIsNotEmptyAfterSavingAndLoadingCheckpointsWithBoundaryValues(t *testing.T) { + // There is a specific scenario in which the weights of the minimum and maximum histogram buckets, + // when saved to a VPACheckpoint and subsequently loaded, result in diminished weights for the minimum buckets. + + // This issue arises due to rounding errors when converting float weights to integers in the VPACheckpoint. + // For instance, consider the weights: + // `w1` which approximates but is slightly larger than or equal to `epsilon`, + // `w2` which approximates but is slightly smaller than or equal to (`MaxCheckpointWeight` * `epsilon`) - `epsilon`. + + // When these weights are stored in a VPACheckpoint, they are translated to integers: + // `w1` rounds to `1` (`wi1`), + // `w2` rounds to `MaxCheckpointWeight` (`wi2`). + + // Upon loading from the VPACheckpoint, the histogram reconstructs its weights using a calculated ratio, + // aimed at reverting integer weights back to float values. This ratio is derived from: + // (`w1` + `w2`) / (`wi1` + `wi2`) + // Reference: https://github.com/plkokanov/autoscaler/blob/2aba67154cd4f117da4702b60a10c38c0651e659/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L256-L269 + + // Given the maximum potential values for `w1`, `w2`, `wi1` and `wi2` we arrive at: + // (`epsilon` + `MaxCheckpointWeight` * `epsilon` - `epsilon`) / (1 + MaxCheckpointWeight) = epsilon * `MaxCheckpointWeight` / (1 + MaxCheckpointWeight) + + // Consequently, the maximum value for this ratio is less than `epsilon`, implying that when `w1`, + // initially scaled to `1`, is adjusted by this ratio, its recalculated weight falls short of `epsilon`. + // The same behavior can be observed when there are more than two weights. + + // This test ensures that in such cases the histogram does not become empty. + // For more information check https://github.com/kubernetes/autoscaler/issues/7726 + histogram := NewHistogram(testHistogramOptions) histogram.AddSample(1, weightEpsilon, anyTime) histogram.AddSample(2, (float64(MaxCheckpointWeight)*weightEpsilon - weightEpsilon), anyTime) From 8892f21919420ca62da8d9551b86e8ecba2dd94c Mon Sep 17 00:00:00 2001 From: Luiz Antonio Date: Mon, 24 Mar 2025 14:57:42 -0400 Subject: [PATCH 28/39] Replace PodResizing with PodResizeInProgress condition in AEP-4016 --- .../enhancements/4016-in-place-updates-support/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md b/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md index f3d56424581..70bf7cd8aef 100644 --- a/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md +++ b/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md @@ -170,7 +170,8 @@ VPA updater will consider that the update failed if: * The pod has condition `PodResizePending` with reason `Infeasible` or * The pod has condition `PodResizePending` with reason `Deferred` and more than 5 minutes elapsed since the update or -* The pod has condition `PodResizing` and more than 1 hour elapsed since the update or +* The pod has condition `PodResizeInProgress` and more than 1 hour elapsed since + the update or * Patch attempt returns an error. Note that in the initial version of In-Place updates, memory limit downscaling will always fail @@ -228,7 +229,7 @@ Today, VPA updater considers the following conditions when deciding if it should * Outside recommended range, * Long-lived pod with significant change. * `EvictionRequirements` are all true. - + `InPlaceOrRecreate` will attempt to apply an update in place if it meets at least one of the following conditions: * Quick OOM, @@ -251,7 +252,7 @@ of the following conditions: The following test scenarios will be added to e2e tests. The `InPlaceOrRecreate` mode will be tested in the following scenarios: -* Admission controller applies recommendation to pod controlled by VPA. +* Admission controller applies recommendation to pod controlled by VPA. * In-place update applied to all containers of a pod. * In-place update will fail. Pod should be evicted and the recommendation applied. * In-place update will fail but `CanEvict` is false, pod should not be evicted. From 4a233bf7df8917aa58834021a86cf85718ae0098 Mon Sep 17 00:00:00 2001 From: Plamen Kokanov Date: Tue, 25 Mar 2025 12:23:00 +0200 Subject: [PATCH 29/39] Address review comments --- .../pkg/recommender/util/histogram_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go index 8691c0557ee..3dfb175b21a 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go @@ -281,13 +281,17 @@ func TestHistogramIsNotEmptyAfterSavingAndLoadingCheckpointsWithBoundaryValues(t // Upon loading from the VPACheckpoint, the histogram reconstructs its weights using a calculated ratio, // aimed at reverting integer weights back to float values. This ratio is derived from: // (`w1` + `w2`) / (`wi1` + `wi2`) - // Reference: https://github.com/plkokanov/autoscaler/blob/2aba67154cd4f117da4702b60a10c38c0651e659/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L256-L269 + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler//pkg/recommender/util/histogram.go#L256-L269 // Given the maximum potential values for `w1`, `w2`, `wi1` and `wi2` we arrive at: // (`epsilon` + `MaxCheckpointWeight` * `epsilon` - `epsilon`) / (1 + MaxCheckpointWeight) = epsilon * `MaxCheckpointWeight` / (1 + MaxCheckpointWeight) // Consequently, the maximum value for this ratio is less than `epsilon`, implying that when `w1`, // initially scaled to `1`, is adjusted by this ratio, its recalculated weight falls short of `epsilon`. + // When the `minBucket`'s weight is less than `epsilon`, the `histogram.IsEmpty()` returns true. + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L181-L183 + // Consequently, the `histogram.Percentile(...)` function will always return 0. + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L159-L162 // The same behavior can be observed when there are more than two weights. // This test ensures that in such cases the histogram does not become empty. From 0c522556c582833dcf73463e9ea266a73450ab37 Mon Sep 17 00:00:00 2001 From: mendelski Date: Mon, 24 Mar 2025 09:27:13 +0000 Subject: [PATCH 30/39] Emit event on successful async scale-up --- .../scaleup/orchestrator/async_initializer.go | 10 +- .../orchestrator/async_initializer_test.go | 109 ++++++++++-------- 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go index e0dd551172c..3cae232ad74 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go @@ -155,9 +155,17 @@ func (s *AsyncNodeGroupInitializer) InitializeNodeGroup(result nodegroups.AsyncN return } klog.Infof("Initial scale-up succeeded. Scale ups: %v", scaleUpInfos) + s.emitScaleUpStatus(&status.ScaleUpStatus{ + Result: status.ScaleUpSuccessful, + ScaleUpInfos: scaleUpInfos, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{result.CreationResult}, + PodsTriggeredScaleUp: s.triggeringPods, + }, nil) } func (s *AsyncNodeGroupInitializer) emitScaleUpStatus(scaleUpStatus *status.ScaleUpStatus, err errors.AutoscalerError) { - status.UpdateScaleUpError(scaleUpStatus, err) + if err != nil { + status.UpdateScaleUpError(scaleUpStatus, err) + } s.scaleUpStatusProcessor.Process(s.context, scaleUpStatus) } diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go index c566328f985..088fc0dd041 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups/asyncnodegroups" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" "k8s.io/autoscaler/cluster-autoscaler/processors/status" processorstest "k8s.io/autoscaler/cluster-autoscaler/processors/test" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" @@ -41,72 +42,80 @@ import ( ) func TestNodePoolAsyncInitialization(t *testing.T) { + scaleUpSize := 3 + failingNodeGroupName := "failing-ng" + provider := testprovider.NewTestCloudProvider( + func(nodeGroup string, increase int) error { + if nodeGroup == failingNodeGroupName { + return fmt.Errorf("Simulated error") + } + return nil + }, nil) + pod := BuildTestPod("p1", 2, 1000) + failingNodeGroup := provider.BuildNodeGroup(failingNodeGroupName, 0, 100, 0, false, true, "T1", nil) + successfulNodeGroup := provider.BuildNodeGroup("async-ng", 0, 100, 0, false, true, "T1", nil) + failedScaleUpErr := errors.ToAutoscalerError(errors.CloudProviderError, fmt.Errorf("Simulated error")).AddPrefix("failed to increase node group size: ") testCases := []struct { - name string - failingScaleUps map[string]bool - expectedScaleUps map[string]int + name string + nodeGroup *testprovider.TestNodeGroup + wantStatus status.ScaleUpStatus }{ { - name: "scale up upcoming node group", - expectedScaleUps: map[string]int{"async-ng": 3}, + name: "scale up upcoming node group", + nodeGroup: successfulNodeGroup, + wantStatus: status.ScaleUpStatus{ + Result: status.ScaleUpSuccessful, + ScaleUpInfos: []nodegroupset.ScaleUpInfo{ + { + Group: successfulNodeGroup, + CurrentSize: 0, + NewSize: scaleUpSize, + MaxSize: successfulNodeGroup.MaxSize(), + }, + }, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{ + {MainCreatedNodeGroup: successfulNodeGroup}, + }, + PodsTriggeredScaleUp: []*apiv1.Pod{pod}, + }, }, { - name: "failing initial scale up", - failingScaleUps: map[string]bool{"async-ng": true}, + name: "failing initial scale up", + nodeGroup: failingNodeGroup, + wantStatus: status.ScaleUpStatus{ + Result: status.ScaleUpError, + ScaleUpError: &failedScaleUpErr, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{ + {MainCreatedNodeGroup: failingNodeGroup}, + }, + FailedResizeNodeGroups: []cloudprovider.NodeGroup{failingNodeGroup}, + PodsTriggeredScaleUp: []*apiv1.Pod{pod}, + }, }, } + listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + upcomingNodeGroup := provider.BuildNodeGroup("upcoming-ng", 0, 100, 0, false, true, "T1", nil) + options := config.AutoscalingOptions{AsyncNodeGroupsEnabled: true} + context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) + assert.NoError(t, err) + option := expander.Option{NodeGroup: upcomingNodeGroup, Pods: []*apiv1.Pod{pod}} + processors := processorstest.NewTestProcessors(&context) + processors.AsyncNodeGroupStateChecker = &asyncnodegroups.MockAsyncNodeGroupStateChecker{IsUpcomingNodeGroup: map[string]bool{upcomingNodeGroup.Id(): true}} + nodeInfo := framework.NewTestNodeInfo(BuildTestNode("t1", 100, 0)) + executor := newScaleUpExecutor(&context, processors.ScaleStateNotifier, processors.AsyncNodeGroupStateChecker) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - scaledUpGroups := make(map[string]int) - provider := testprovider.NewTestCloudProvider( - func(nodeGroup string, increase int) error { - if tc.failingScaleUps[nodeGroup] { - return fmt.Errorf("Simulated error") - } - scaledUpGroups[nodeGroup] += increase - return nil - }, nil) - options := config.AutoscalingOptions{ - AsyncNodeGroupsEnabled: true, - } - listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) - context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) - assert.NoError(t, err) - p1 := BuildTestPod("p1", 2, 1000) - upcomingNodeGroup := provider.BuildNodeGroup("upcoming-ng", 0, 100, 0, false, true, "T1", nil) - createdNodeGroup := provider.BuildNodeGroup("async-ng", 0, 100, 0, false, true, "T1", nil) - option := expander.Option{ - NodeGroup: upcomingNodeGroup, - Pods: []*apiv1.Pod{p1}, - } - processors := processorstest.NewTestProcessors(&context) - processors.AsyncNodeGroupStateChecker = &asyncnodegroups.MockAsyncNodeGroupStateChecker{IsUpcomingNodeGroup: map[string]bool{upcomingNodeGroup.Id(): true}} - nodeInfo := framework.NewTestNodeInfo(BuildTestNode("t1", 100, 0)) - executor := newScaleUpExecutor(&context, processors.ScaleStateNotifier, processors.AsyncNodeGroupStateChecker) scaleUpStatusProcessor := &fakeScaleUpStatusProcessor{} initializer := NewAsyncNodeGroupInitializer(&option, nodeInfo, executor, taints.TaintConfig{}, nil, scaleUpStatusProcessor, &context, false) - initializer.SetTargetSize(upcomingNodeGroup.Id(), 3) + initializer.SetTargetSize(upcomingNodeGroup.Id(), int64(scaleUpSize)) asyncResult := nodegroups.AsyncNodeGroupCreationResult{ - CreationResult: nodegroups.CreateNodeGroupResult{MainCreatedNodeGroup: createdNodeGroup}, + CreationResult: nodegroups.CreateNodeGroupResult{MainCreatedNodeGroup: tc.nodeGroup}, CreatedToUpcomingMapping: map[string]string{ - createdNodeGroup.Id(): upcomingNodeGroup.Id(), + tc.nodeGroup.Id(): upcomingNodeGroup.Id(), }, } initializer.InitializeNodeGroup(asyncResult) - assert.Equal(t, len(scaledUpGroups), len(tc.expectedScaleUps)) - for groupName, increase := range tc.expectedScaleUps { - assert.Equal(t, increase, scaledUpGroups[groupName]) - } - if len(tc.failingScaleUps) > 0 { - expectedErr := errors.ToAutoscalerError(errors.CloudProviderError, fmt.Errorf("Simulated error")).AddPrefix("failed to increase node group size: ") - assert.Equal(t, scaleUpStatusProcessor.lastStatus, &status.ScaleUpStatus{ - Result: status.ScaleUpError, - ScaleUpError: &expectedErr, - CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{asyncResult.CreationResult}, - FailedResizeNodeGroups: []cloudprovider.NodeGroup{createdNodeGroup}, - PodsTriggeredScaleUp: option.Pods, - }) - } + assert.Equal(t, *scaleUpStatusProcessor.lastStatus, tc.wantStatus) }) } } From 5e1fc195a318587b830b456b3d6d056a4b375eef Mon Sep 17 00:00:00 2001 From: elmiko Date: Tue, 25 Mar 2025 14:25:20 -0400 Subject: [PATCH 31/39] refactor findScalableResourceProviderIDs in clusterapi this change refactors the function so that it each distinct machine state can be filtered more easily. the unit tests have been supplemented, but not changed to ensure that the functionality continues to work as expected. these changes are to help better detect edge cases where machines can be transiting through pending phase and might be removed by the autoscaler. --- .../clusterapi/clusterapi_controller.go | 69 ++++++++++++------- .../clusterapi/clusterapi_controller_test.go | 13 ---- .../clusterapi/clusterapi_nodegroup_test.go | 54 ++++++++++++--- 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index 6c95ca23fc0..bf6cca92267 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -638,6 +638,7 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un } for _, machine := range machines { + // Failed Machines // In some cases it is possible for a machine to have acquired a provider ID from the infrastructure and // then become failed later. We want to ensure that a failed machine is not counted towards the total // number of nodes in the cluster, for this reason we will detect a failed machine first, regardless @@ -648,20 +649,53 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un } if found { - klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) - // Provide a fake ID to allow the autoscaler to track machines that will never + // Provide a normalized ID to allow the autoscaler to track machines that will never // become nodes and mark the nodegroup unhealthy after maxNodeProvisionTime. // Fake ID needs to be recognised later and converted into a machine key. // Use an underscore as a separator between namespace and name as it is not a // valid character within a namespace name. + klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) providerIDs = append(providerIDs, createFailedMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName())) continue } - // Next we check for the provider ID. Machines with a provider ID can fall into one of a few - // categories: creating, running, deleting, and sometimes failed (but we filtered those earlier). - // Depending on which category the machine is in, it might have a deleting or pending guard - // added to it's provider ID so that we can later properly filter machines. + // Deleting Machines + // Machines that are in deleting state should be identified so that in scenarios where the core + // autoscaler would like to adjust the size of a node group, we can give a proper count and + // be able to filter machines in that state, regardless of whether they are still active nodes in the cluster. + // We give these machines normalized provider IDs to aid in the filtering process. + if !machine.GetDeletionTimestamp().IsZero() { + klog.V(4).Infof("Machine %q has a non-zero deletion timestamp", machine.GetName()) + providerIDs = append(providerIDs, createDeletingMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName())) + continue + } + + // Pending Machines + // Machines that do not yet have an associated node reference are considering to be pending. These + // nodes need to be filtered so that in a case where a machine is not becoming a node, or the instance + // lifecycle has changed during provisioning (eg spot instance going away), or the core autoscaler has + // decided that the node is not needed. + // Look for a node reference in the status, a machine without a node reference, and that is also not + // in failed or deleting state, has not yet become a node, and should be marked as pending. + // We give these machines normalized provider IDs to aid in the filtering process. + _, found, err = unstructured.NestedFieldCopy(machine.UnstructuredContent(), "status", "nodeRef") + if err != nil { + return nil, err + } + + if !found { + klog.V(4).Infof("Status.NodeRef of machine %q is currently nil", machine.GetName()) + providerIDs = append(providerIDs, createPendingMachineProviderID(machine.GetNamespace(), machine.GetName())) + continue + } + + // Running Machines + // We have filtered out the machines in failed, deleting, and pending states. We now check the provider + // ID and potentially the node reference details. It is ok for a machine not to have a provider ID as + // not all CAPI provider implement this field, but a machine in running state should have a valid + // node reference. If a provider ID is present, we add that to the list as we know it is not failed, + // deleting, or pending. If an empty provider ID is present, we check the node details to ensure that + // the machine references a valid node. providerID, found, err := unstructured.NestedString(machine.UnstructuredContent(), "spec", "providerID") if err != nil { return nil, err @@ -669,12 +703,7 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un if found { if providerID != "" { - // If the machine is deleting, prepend the deletion guard on the provider id - // so that it can be properly filtered when counting the number of nodes and instances. - if !machine.GetDeletionTimestamp().IsZero() { - klog.V(4).Infof("Machine %q has a non-zero deletion timestamp", machine.GetName()) - providerID = createDeletingMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName()) - } + // Machine has a provider ID, add it to the list providerIDs = append(providerIDs, providerID) continue } @@ -682,19 +711,7 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un klog.Warningf("Machine %q has no providerID", machine.GetName()) - // Look for a node reference in the status, a machine with a provider ID but no node reference has not - // yet become a node and should be marked as pending. - _, found, err = unstructured.NestedFieldCopy(machine.UnstructuredContent(), "status", "nodeRef") - if err != nil { - return nil, err - } - - if !found { - klog.V(4).Infof("Status.NodeRef of machine %q is currently nil", machine.GetName()) - providerIDs = append(providerIDs, createPendingMachineProviderID(machine.GetNamespace(), machine.GetName())) - continue - } - + // Begin checking to determine if the node reference is valid nodeRefKind, found, err := unstructured.NestedString(machine.UnstructuredContent(), "status", "nodeRef", "kind") if err != nil { return nil, err @@ -716,6 +733,8 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un return nil, fmt.Errorf("unknown node %q", nodeRefName) } + // A node has been found that corresponds to this machine, since we know that this machine has + // an empty provider ID, we add the provider ID from the node to the list. if node != nil { providerIDs = append(providerIDs, node.Spec.ProviderID) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index 501f89da17d..fadc07aa783 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -1335,19 +1335,6 @@ func TestControllerMachineSetNodeNamesUsingProviderID(t *testing.T) { controller, stop := mustCreateTestController(t, testConfig) defer stop() - // Remove Status.NodeRef.Name on all the machines. We want to - // force machineSetNodeNames() to only consider the provider - // ID for lookups. - for i := range testConfig.machines { - machine := testConfig.machines[i].DeepCopy() - - unstructured.RemoveNestedField(machine.Object, "status", "nodeRef") - - if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { - t.Fatalf("unexpected error updating machine, got %v", err) - } - } - nodegroups, err := controller.nodeGroups() if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index 11bcec4e3df..75ec29805ba 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -406,16 +406,18 @@ func TestNodeGroupIncreaseSize(t *testing.T) { func TestNodeGroupDecreaseTargetSize(t *testing.T) { type testCase struct { - description string - delta int - initial int32 - targetSizeIncrement int32 - expected int32 - expectedError bool - includeDeletingMachine bool - includeFailedMachine bool - includeFailedMachineWithProviderID bool - includePendingMachine bool + description string + delta int + initial int32 + targetSizeIncrement int32 + expected int32 + expectedError bool + includeDeletingMachine bool + includeFailedMachine bool + includeFailedMachineWithProviderID bool + includePendingMachine bool + includePendingMachineWithProviderID bool + machinesDoNotHaveProviderIDs bool } test := func(t *testing.T, tc *testCase, testConfig *testConfig) { @@ -468,7 +470,9 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { // Simulate a pending machine machine := testConfig.machines[2].DeepCopy() - unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + if !tc.includePendingMachineWithProviderID { + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + } unstructured.RemoveNestedField(machine.Object, "status", "nodeRef") if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { @@ -476,6 +480,17 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { } } + // machines with no provider id can be created on some providers, notably bare metal + if tc.machinesDoNotHaveProviderIDs { + for _, machine := range testConfig.machines { + updated := machine.DeepCopy() + unstructured.RemoveNestedField(updated.Object, "spec", "providerID") + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, updated); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + } + nodegroups, err := controller.nodeGroups() if err != nil { t.Fatalf("unexpected error: %v", err) @@ -648,6 +663,23 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { includeFailedMachine: true, includeFailedMachineWithProviderID: true, }, + { + description: "A node group with 4 replicas with one pending machine that has a provider ID should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includePendingMachine: true, + includePendingMachineWithProviderID: true, + }, + { + description: "A node group with target size 4 but only 3 existing instances without provider IDs should not scale out", + initial: 3, + targetSizeIncrement: 1, + expected: 3, + delta: -1, + machinesDoNotHaveProviderIDs: true, + }, } annotations := map[string]string{ From e713b51bd6fb48ecbbf4ee98cfc022807c3fbe40 Mon Sep 17 00:00:00 2001 From: KrJin <33444112+jincong8973@users.noreply.github.com> Date: Tue, 25 Mar 2025 20:32:49 +0800 Subject: [PATCH 32/39] feat: add missing field zeroOrMaxNodeScaling and ignoreDaemonSetsUtilization to NodeGroupAutoscalingOptions [squashed]Add field IgnoreDaemonSetsUtilization and zeroOrMaxNodeScaling that missing in externalgrpc proto --- .../cloudprovider/externalgrpc/README.md | 7 +- .../wrapper/wrapper.go | 8 +- .../externalgrpc/externalgrpc_node_group.go | 4 + .../externalgrpc_node_group_test.go | 49 +- .../externalgrpc/protos/externalgrpc.pb.go | 1663 +++++------------ .../externalgrpc/protos/externalgrpc.proto | 84 +- .../protos/externalgrpc_grpc.pb.go | 72 +- 7 files changed, 565 insertions(+), 1322 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/README.md b/cluster-autoscaler/cloudprovider/externalgrpc/README.md index c058e8c5bfe..d73118fc71f 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/README.md +++ b/cluster-autoscaler/cloudprovider/externalgrpc/README.md @@ -61,7 +61,12 @@ go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3 ``` -2. generate gRPC client and server code: +2. import proto dependencies using go modules +```bash +go mod vendor +``` + +3. generate gRPC client and server code: ```bash protoc \ diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go index ae57430d962..45bbad632c5 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go @@ -103,7 +103,7 @@ func (w *Wrapper) NodeGroupForNode(_ context.Context, req *protos.NodeGroupForNo // Checks if ng is nil interface or contains nil value if ng == nil || reflect.ValueOf(ng).IsNil() { return &protos.NodeGroupForNodeResponse{ - NodeGroup: &protos.NodeGroup{}, //NodeGroup with id = "", meaning the node should not be processed by cluster autoscaler + NodeGroup: &protos.NodeGroup{}, // NodeGroup with id = "", meaning the node should not be processed by cluster autoscaler }, nil } return &protos.NodeGroupForNodeResponse{ @@ -365,6 +365,8 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu ScaleDownUnneededTime: pbDefaults.GetScaleDownUnneededTime().Duration, ScaleDownUnreadyTime: pbDefaults.GetScaleDownUnneededTime().Duration, MaxNodeProvisionTime: pbDefaults.GetMaxNodeProvisionTime().Duration, + ZeroOrMaxNodeScaling: pbDefaults.GetZeroOrMaxNodeScaling(), + IgnoreDaemonSetsUtilization: pbDefaults.GetIgnoreDaemonSetsUtilization(), } opts, err := ng.GetOptions(defaults) if err != nil { @@ -374,7 +376,7 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu return nil, err } if opts == nil { - return nil, fmt.Errorf("GetOptions not implemented") //make this explicitly so that grpc response is discarded + return nil, fmt.Errorf("GetOptions not implemented") // make this explicitly so that grpc response is discarded } return &protos.NodeGroupAutoscalingOptionsResponse{ NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ @@ -389,6 +391,8 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu MaxNodeProvisionTime: &metav1.Duration{ Duration: opts.MaxNodeProvisionTime, }, + ZeroOrMaxNodeScaling: opts.ZeroOrMaxNodeScaling, + IgnoreDaemonSetsUtilization: opts.IgnoreDaemonSetsUtilization, }, }, nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go index 2f8000aded1..f62f3133eaa 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go @@ -283,6 +283,8 @@ func (n *NodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*co MaxNodeProvisionTime: &metav1.Duration{ Duration: defaults.MaxNodeProvisionTime, }, + ZeroOrMaxNodeScaling: defaults.ZeroOrMaxNodeScaling, + IgnoreDaemonSetsUtilization: defaults.IgnoreDaemonSetsUtilization, }, }) if err != nil { @@ -303,6 +305,8 @@ func (n *NodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*co ScaleDownUnneededTime: pbOpts.GetScaleDownUnneededTime().Duration, ScaleDownUnreadyTime: pbOpts.GetScaleDownUnreadyTime().Duration, MaxNodeProvisionTime: pbOpts.GetMaxNodeProvisionTime().Duration, + ZeroOrMaxNodeScaling: pbOpts.GetZeroOrMaxNodeScaling(), + IgnoreDaemonSetsUtilization: pbOpts.GetIgnoreDaemonSetsUtilization(), } return opts, nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go index 7dc27a4eac6..d4544e12d9c 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go @@ -244,7 +244,7 @@ func TestCloudProvider_GetOptions(t *testing.T) { client, m, teardown := setupTest(t) defer teardown() - // test correct call + // test correct call, NodeGroupAutoscalingOptionsResponse will override default options m.On( "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { return req.Id == "nodeGroup1" @@ -257,6 +257,8 @@ func TestCloudProvider_GetOptions(t *testing.T) { ScaleDownUnneededTime: &v1.Duration{Duration: time.Minute}, ScaleDownUnreadyTime: &v1.Duration{Duration: time.Hour}, MaxNodeProvisionTime: &v1.Duration{Duration: time.Minute}, + ZeroOrMaxNodeScaling: true, + IgnoreDaemonSetsUtilization: true, }, }, nil, @@ -267,12 +269,15 @@ func TestCloudProvider_GetOptions(t *testing.T) { client: client, grpcTimeout: defaultGRPCTimeout, } + defaultsOpts := config.NodeGroupAutoscalingOptions{ ScaleDownUtilizationThreshold: 0.6, ScaleDownGpuUtilizationThreshold: 0.7, ScaleDownUnneededTime: time.Minute, ScaleDownUnreadyTime: time.Hour, MaxNodeProvisionTime: time.Minute, + ZeroOrMaxNodeScaling: false, + IgnoreDaemonSetsUtilization: false, } opts, err := ng1.GetOptions(defaultsOpts) @@ -282,6 +287,8 @@ func TestCloudProvider_GetOptions(t *testing.T) { assert.Equal(t, time.Minute, opts.ScaleDownUnneededTime) assert.Equal(t, time.Hour, opts.ScaleDownUnreadyTime) assert.Equal(t, time.Minute, opts.MaxNodeProvisionTime) + assert.Equal(t, true, opts.ZeroOrMaxNodeScaling) + assert.Equal(t, true, opts.IgnoreDaemonSetsUtilization) // test grpc error m.On( @@ -289,9 +296,11 @@ func TestCloudProvider_GetOptions(t *testing.T) { return req.Id == "nodeGroup2" }), ).Return( - &protos.NodeGroupAutoscalingOptionsResponse{}, + &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{}, + }, fmt.Errorf("mock error"), - ) + ).Once() ng2 := NodeGroup{ id: "nodeGroup2", @@ -343,6 +352,40 @@ func TestCloudProvider_GetOptions(t *testing.T) { assert.Error(t, err) assert.Equal(t, cloudprovider.ErrNotImplemented, err) + // test with default options + m.On( + "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { + return req.Id == "nodeGroup5" + }), + ).Return( + &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: 0.6, + ScaleDownGpuUtilizationThreshold: 0.7, + ScaleDownUnneededTime: &v1.Duration{Duration: time.Minute}, + ScaleDownUnreadyTime: &v1.Duration{Duration: time.Hour}, + MaxNodeProvisionTime: &v1.Duration{Duration: time.Minute}, + }, + }, + nil, + ) + + ng5 := NodeGroup{ + id: "nodeGroup5", + client: client, + grpcTimeout: defaultGRPCTimeout, + } + + opts, err = ng5.GetOptions(defaultsOpts) + assert.NoError(t, err) + assert.Equal(t, 0.6, opts.ScaleDownUtilizationThreshold) + assert.Equal(t, 0.7, opts.ScaleDownGpuUtilizationThreshold) + assert.Equal(t, time.Minute, opts.ScaleDownUnneededTime) + assert.Equal(t, time.Hour, opts.ScaleDownUnreadyTime) + assert.Equal(t, time.Minute, opts.MaxNodeProvisionTime) + assert.Equal(t, false, opts.ZeroOrMaxNodeScaling) + assert.Equal(t, false, opts.IgnoreDaemonSetsUtilization) + } func TestCloudProvider_TargetSize(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go index 6bc0ceb3667..c500ba41f11 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go @@ -15,8 +15,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v3.21.12 +// protoc-gen-go v1.36.6 +// protoc v5.29.2 // source: cloudprovider/externalgrpc/protos/externalgrpc.proto package protos @@ -30,6 +30,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -96,10 +97,7 @@ func (InstanceStatus_InstanceState) EnumDescriptor() ([]byte, []int) { } type NodeGroup struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group on the cloud provider. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // MinSize of the node group on the cloud provider. @@ -107,16 +105,16 @@ type NodeGroup struct { // MaxSize of the node group on the cloud provider. MaxSize int32 `protobuf:"varint,3,opt,name=maxSize,proto3" json:"maxSize,omitempty"` // Debug returns a string containing all information regarding this node group. - Debug string `protobuf:"bytes,4,opt,name=debug,proto3" json:"debug,omitempty"` + Debug string `protobuf:"bytes,4,opt,name=debug,proto3" json:"debug,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroup) Reset() { *x = NodeGroup{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroup) String() string { @@ -127,7 +125,7 @@ func (*NodeGroup) ProtoMessage() {} func (x *NodeGroup) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -171,27 +169,24 @@ func (x *NodeGroup) GetDebug() string { } type ExternalGrpcNode struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node assigned by the cloud provider in the format: ://. ProviderID string `protobuf:"bytes,1,opt,name=providerID,proto3" json:"providerID,omitempty"` // Name of the node assigned by the cloud provider. Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // labels is a map of {key,value} pairs with the node's labels. - Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // If specified, the node's annotations. - Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExternalGrpcNode) Reset() { *x = ExternalGrpcNode{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExternalGrpcNode) String() string { @@ -202,7 +197,7 @@ func (*ExternalGrpcNode) ProtoMessage() {} func (x *ExternalGrpcNode) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -246,18 +241,16 @@ func (x *ExternalGrpcNode) GetAnnotations() map[string]string { } type NodeGroupsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupsRequest) Reset() { *x = NodeGroupsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupsRequest) String() string { @@ -268,7 +261,7 @@ func (*NodeGroupsRequest) ProtoMessage() {} func (x *NodeGroupsRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -284,21 +277,18 @@ func (*NodeGroupsRequest) Descriptor() ([]byte, []int) { } type NodeGroupsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // All the node groups that the cloud provider service supports. - NodeGroups []*NodeGroup `protobuf:"bytes,1,rep,name=nodeGroups,proto3" json:"nodeGroups,omitempty"` + NodeGroups []*NodeGroup `protobuf:"bytes,1,rep,name=nodeGroups,proto3" json:"nodeGroups,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupsResponse) Reset() { *x = NodeGroupsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupsResponse) String() string { @@ -309,7 +299,7 @@ func (*NodeGroupsResponse) ProtoMessage() {} func (x *NodeGroupsResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -332,21 +322,18 @@ func (x *NodeGroupsResponse) GetNodeGroups() []*NodeGroup { } type NodeGroupForNodeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node for which the request is performed. - Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupForNodeRequest) Reset() { *x = NodeGroupForNodeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupForNodeRequest) String() string { @@ -357,7 +344,7 @@ func (*NodeGroupForNodeRequest) ProtoMessage() {} func (x *NodeGroupForNodeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -380,21 +367,18 @@ func (x *NodeGroupForNodeRequest) GetNode() *ExternalGrpcNode { } type NodeGroupForNodeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node group for the given node. nodeGroup with id = "" means no node group. - NodeGroup *NodeGroup `protobuf:"bytes,1,opt,name=nodeGroup,proto3" json:"nodeGroup,omitempty"` + NodeGroup *NodeGroup `protobuf:"bytes,1,opt,name=nodeGroup,proto3" json:"nodeGroup,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupForNodeResponse) Reset() { *x = NodeGroupForNodeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupForNodeResponse) String() string { @@ -405,7 +389,7 @@ func (*NodeGroupForNodeResponse) ProtoMessage() {} func (x *NodeGroupForNodeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -428,25 +412,22 @@ func (x *NodeGroupForNodeResponse) GetNodeGroup() *NodeGroup { } type PricingNodePriceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node for which the request is performed. Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` // Start time for the request period. StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` // End time for the request period. - EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingNodePriceRequest) Reset() { *x = PricingNodePriceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingNodePriceRequest) String() string { @@ -457,7 +438,7 @@ func (*PricingNodePriceRequest) ProtoMessage() {} func (x *PricingNodePriceRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -494,21 +475,18 @@ func (x *PricingNodePriceRequest) GetEndTime() *v1.Time { } type PricingNodePriceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Theoretical minimum price of running a node for a given period. - Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingNodePriceResponse) Reset() { *x = PricingNodePriceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingNodePriceResponse) String() string { @@ -519,7 +497,7 @@ func (*PricingNodePriceResponse) ProtoMessage() {} func (x *PricingNodePriceResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -542,25 +520,22 @@ func (x *PricingNodePriceResponse) GetPrice() float64 { } type PricingPodPriceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Pod for which the request is performed. Pod *v11.Pod `protobuf:"bytes,1,opt,name=pod,proto3" json:"pod,omitempty"` // Start time for the request period. StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` // End time for the request period. - EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingPodPriceRequest) Reset() { *x = PricingPodPriceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingPodPriceRequest) String() string { @@ -571,7 +546,7 @@ func (*PricingPodPriceRequest) ProtoMessage() {} func (x *PricingPodPriceRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -608,21 +583,18 @@ func (x *PricingPodPriceRequest) GetEndTime() *v1.Time { } type PricingPodPriceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Theoretical minimum price of running a pod for a given period. - Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingPodPriceResponse) Reset() { *x = PricingPodPriceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingPodPriceResponse) String() string { @@ -633,7 +605,7 @@ func (*PricingPodPriceResponse) ProtoMessage() {} func (x *PricingPodPriceResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -656,18 +628,16 @@ func (x *PricingPodPriceResponse) GetPrice() float64 { } type GPULabelRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPULabelRequest) Reset() { *x = GPULabelRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GPULabelRequest) String() string { @@ -678,7 +648,7 @@ func (*GPULabelRequest) ProtoMessage() {} func (x *GPULabelRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -694,21 +664,18 @@ func (*GPULabelRequest) Descriptor() ([]byte, []int) { } type GPULabelResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Label added to nodes with a GPU resource. - Label string `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"` + Label string `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPULabelResponse) Reset() { *x = GPULabelResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GPULabelResponse) String() string { @@ -719,7 +686,7 @@ func (*GPULabelResponse) ProtoMessage() {} func (x *GPULabelResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -742,18 +709,16 @@ func (x *GPULabelResponse) GetLabel() string { } type GetAvailableGPUTypesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetAvailableGPUTypesRequest) Reset() { *x = GetAvailableGPUTypesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetAvailableGPUTypesRequest) String() string { @@ -764,7 +729,7 @@ func (*GetAvailableGPUTypesRequest) ProtoMessage() {} func (x *GetAvailableGPUTypesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -780,21 +745,18 @@ func (*GetAvailableGPUTypesRequest) Descriptor() ([]byte, []int) { } type GetAvailableGPUTypesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // GPU types passed in as opaque key-value pairs. - GpuTypes map[string]*anypb.Any `protobuf:"bytes,1,rep,name=gpuTypes,proto3" json:"gpuTypes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + GpuTypes map[string]*anypb.Any `protobuf:"bytes,1,rep,name=gpuTypes,proto3" json:"gpuTypes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetAvailableGPUTypesResponse) Reset() { *x = GetAvailableGPUTypesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetAvailableGPUTypesResponse) String() string { @@ -805,7 +767,7 @@ func (*GetAvailableGPUTypesResponse) ProtoMessage() {} func (x *GetAvailableGPUTypesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -828,18 +790,16 @@ func (x *GetAvailableGPUTypesResponse) GetGpuTypes() map[string]*anypb.Any { } type CleanupRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CleanupRequest) Reset() { *x = CleanupRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CleanupRequest) String() string { @@ -850,7 +810,7 @@ func (*CleanupRequest) ProtoMessage() {} func (x *CleanupRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -866,18 +826,16 @@ func (*CleanupRequest) Descriptor() ([]byte, []int) { } type CleanupResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CleanupResponse) Reset() { *x = CleanupResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CleanupResponse) String() string { @@ -888,7 +846,7 @@ func (*CleanupResponse) ProtoMessage() {} func (x *CleanupResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -904,18 +862,16 @@ func (*CleanupResponse) Descriptor() ([]byte, []int) { } type RefreshRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RefreshRequest) Reset() { *x = RefreshRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RefreshRequest) String() string { @@ -926,7 +882,7 @@ func (*RefreshRequest) ProtoMessage() {} func (x *RefreshRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -942,18 +898,16 @@ func (*RefreshRequest) Descriptor() ([]byte, []int) { } type RefreshResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RefreshResponse) Reset() { *x = RefreshResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RefreshResponse) String() string { @@ -964,7 +918,7 @@ func (*RefreshResponse) ProtoMessage() {} func (x *RefreshResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -980,21 +934,18 @@ func (*RefreshResponse) Descriptor() ([]byte, []int) { } type NodeGroupTargetSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTargetSizeRequest) Reset() { *x = NodeGroupTargetSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTargetSizeRequest) String() string { @@ -1005,7 +956,7 @@ func (*NodeGroupTargetSizeRequest) ProtoMessage() {} func (x *NodeGroupTargetSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1028,21 +979,18 @@ func (x *NodeGroupTargetSizeRequest) GetId() string { } type NodeGroupTargetSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Current target size of the node group. - TargetSize int32 `protobuf:"varint,1,opt,name=targetSize,proto3" json:"targetSize,omitempty"` + TargetSize int32 `protobuf:"varint,1,opt,name=targetSize,proto3" json:"targetSize,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTargetSizeResponse) Reset() { *x = NodeGroupTargetSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTargetSizeResponse) String() string { @@ -1053,7 +1001,7 @@ func (*NodeGroupTargetSizeResponse) ProtoMessage() {} func (x *NodeGroupTargetSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1076,23 +1024,20 @@ func (x *NodeGroupTargetSizeResponse) GetTargetSize() int32 { } type NodeGroupIncreaseSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Number of nodes to add. Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupIncreaseSizeRequest) Reset() { *x = NodeGroupIncreaseSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupIncreaseSizeRequest) String() string { @@ -1103,7 +1048,7 @@ func (*NodeGroupIncreaseSizeRequest) ProtoMessage() {} func (x *NodeGroupIncreaseSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1133,18 +1078,16 @@ func (x *NodeGroupIncreaseSizeRequest) GetId() string { } type NodeGroupIncreaseSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupIncreaseSizeResponse) Reset() { *x = NodeGroupIncreaseSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupIncreaseSizeResponse) String() string { @@ -1155,7 +1098,7 @@ func (*NodeGroupIncreaseSizeResponse) ProtoMessage() {} func (x *NodeGroupIncreaseSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1171,23 +1114,20 @@ func (*NodeGroupIncreaseSizeResponse) Descriptor() ([]byte, []int) { } type NodeGroupDeleteNodesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // List of nodes to delete. Nodes []*ExternalGrpcNode `protobuf:"bytes,1,rep,name=nodes,proto3" json:"nodes,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDeleteNodesRequest) Reset() { *x = NodeGroupDeleteNodesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDeleteNodesRequest) String() string { @@ -1198,7 +1138,7 @@ func (*NodeGroupDeleteNodesRequest) ProtoMessage() {} func (x *NodeGroupDeleteNodesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1228,18 +1168,16 @@ func (x *NodeGroupDeleteNodesRequest) GetId() string { } type NodeGroupDeleteNodesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDeleteNodesResponse) Reset() { *x = NodeGroupDeleteNodesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDeleteNodesResponse) String() string { @@ -1250,7 +1188,7 @@ func (*NodeGroupDeleteNodesResponse) ProtoMessage() {} func (x *NodeGroupDeleteNodesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1266,23 +1204,20 @@ func (*NodeGroupDeleteNodesResponse) Descriptor() ([]byte, []int) { } type NodeGroupDecreaseTargetSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Number of nodes to delete. Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDecreaseTargetSizeRequest) Reset() { *x = NodeGroupDecreaseTargetSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDecreaseTargetSizeRequest) String() string { @@ -1293,7 +1228,7 @@ func (*NodeGroupDecreaseTargetSizeRequest) ProtoMessage() {} func (x *NodeGroupDecreaseTargetSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1323,18 +1258,16 @@ func (x *NodeGroupDecreaseTargetSizeRequest) GetId() string { } type NodeGroupDecreaseTargetSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDecreaseTargetSizeResponse) Reset() { *x = NodeGroupDecreaseTargetSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDecreaseTargetSizeResponse) String() string { @@ -1345,7 +1278,7 @@ func (*NodeGroupDecreaseTargetSizeResponse) ProtoMessage() {} func (x *NodeGroupDecreaseTargetSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1361,21 +1294,18 @@ func (*NodeGroupDecreaseTargetSizeResponse) Descriptor() ([]byte, []int) { } type NodeGroupNodesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupNodesRequest) Reset() { *x = NodeGroupNodesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupNodesRequest) String() string { @@ -1386,7 +1316,7 @@ func (*NodeGroupNodesRequest) ProtoMessage() {} func (x *NodeGroupNodesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1409,21 +1339,18 @@ func (x *NodeGroupNodesRequest) GetId() string { } type NodeGroupNodesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // list of cloud provider instances in a node group. - Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` + Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupNodesResponse) Reset() { *x = NodeGroupNodesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupNodesResponse) String() string { @@ -1434,7 +1361,7 @@ func (*NodeGroupNodesResponse) ProtoMessage() {} func (x *NodeGroupNodesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1457,23 +1384,20 @@ func (x *NodeGroupNodesResponse) GetInstances() []*Instance { } type Instance struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Id of the instance. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // Status of the node. - Status *InstanceStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + Status *InstanceStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Instance) Reset() { *x = Instance{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Instance) String() string { @@ -1484,7 +1408,7 @@ func (*Instance) ProtoMessage() {} func (x *Instance) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1515,24 +1439,21 @@ func (x *Instance) GetStatus() *InstanceStatus { // InstanceStatus represents the instance status. type InstanceStatus struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // InstanceState tells if the instance is running, being created or being deleted. InstanceState InstanceStatus_InstanceState `protobuf:"varint,1,opt,name=instanceState,proto3,enum=clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus_InstanceState" json:"instanceState,omitempty"` // ErrorInfo provides information about the error status. // If there is no error condition related to instance, then errorInfo.errorCode should be an empty string. - ErrorInfo *InstanceErrorInfo `protobuf:"bytes,2,opt,name=errorInfo,proto3" json:"errorInfo,omitempty"` + ErrorInfo *InstanceErrorInfo `protobuf:"bytes,2,opt,name=errorInfo,proto3" json:"errorInfo,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceStatus) Reset() { *x = InstanceStatus{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InstanceStatus) String() string { @@ -1543,7 +1464,7 @@ func (*InstanceStatus) ProtoMessage() {} func (x *InstanceStatus) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1574,10 +1495,7 @@ func (x *InstanceStatus) GetErrorInfo() *InstanceErrorInfo { // InstanceErrorInfo provides information about error condition on instance. type InstanceErrorInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ErrorCode is cloud-provider specific error code for error condition. // An empty string for errorCode means there is no errorInfo for the instance (nil). ErrorCode string `protobuf:"bytes,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"` @@ -1585,15 +1503,15 @@ type InstanceErrorInfo struct { ErrorMessage string `protobuf:"bytes,2,opt,name=errorMessage,proto3" json:"errorMessage,omitempty"` // InstanceErrorClass defines the class of error condition. InstanceErrorClass int32 `protobuf:"varint,3,opt,name=instanceErrorClass,proto3" json:"instanceErrorClass,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceErrorInfo) Reset() { *x = InstanceErrorInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InstanceErrorInfo) String() string { @@ -1604,7 +1522,7 @@ func (*InstanceErrorInfo) ProtoMessage() {} func (x *InstanceErrorInfo) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1641,21 +1559,18 @@ func (x *InstanceErrorInfo) GetInstanceErrorClass() int32 { } type NodeGroupTemplateNodeInfoRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTemplateNodeInfoRequest) Reset() { *x = NodeGroupTemplateNodeInfoRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTemplateNodeInfoRequest) String() string { @@ -1666,7 +1581,7 @@ func (*NodeGroupTemplateNodeInfoRequest) ProtoMessage() {} func (x *NodeGroupTemplateNodeInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1689,21 +1604,18 @@ func (x *NodeGroupTemplateNodeInfoRequest) GetId() string { } type NodeGroupTemplateNodeInfoResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // nodeInfo is the extracted data from the cloud provider, as a primitive Kubernetes Node type. - NodeInfo *v11.Node `protobuf:"bytes,1,opt,name=nodeInfo,proto3" json:"nodeInfo,omitempty"` + NodeInfo *v11.Node `protobuf:"bytes,1,opt,name=nodeInfo,proto3" json:"nodeInfo,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTemplateNodeInfoResponse) Reset() { *x = NodeGroupTemplateNodeInfoResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTemplateNodeInfoResponse) String() string { @@ -1714,7 +1626,7 @@ func (*NodeGroupTemplateNodeInfoResponse) ProtoMessage() {} func (x *NodeGroupTemplateNodeInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1737,10 +1649,7 @@ func (x *NodeGroupTemplateNodeInfoResponse) GetNodeInfo() *v11.Node { } type NodeGroupAutoscalingOptions struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ScaleDownUtilizationThreshold sets threshold for nodes to be considered for scale down // if cpu or memory utilization is over threshold. ScaleDownUtilizationThreshold float64 `protobuf:"fixed64,1,opt,name=scaleDownUtilizationThreshold,proto3" json:"scaleDownUtilizationThreshold,omitempty"` @@ -1755,15 +1664,19 @@ type NodeGroupAutoscalingOptions struct { ScaleDownUnreadyTime *v1.Duration `protobuf:"bytes,4,opt,name=scaleDownUnreadyTime,proto3" json:"scaleDownUnreadyTime,omitempty"` // MaxNodeProvisionTime time CA waits for node to be provisioned MaxNodeProvisionTime *v1.Duration `protobuf:"bytes,5,opt,name=MaxNodeProvisionTime,proto3" json:"MaxNodeProvisionTime,omitempty"` + // ZeroOrMaxNodeScaling means that a node group should be scaled up to maximum size or down to zero nodes all at once instead of one-by-one. + ZeroOrMaxNodeScaling bool `protobuf:"varint,6,opt,name=zeroOrMaxNodeScaling,proto3" json:"zeroOrMaxNodeScaling,omitempty"` + // IgnoreDaemonSetsUtilization sets if daemonsets utilization should be considered during node scale-down + IgnoreDaemonSetsUtilization bool `protobuf:"varint,7,opt,name=ignoreDaemonSetsUtilization,proto3" json:"ignoreDaemonSetsUtilization,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptions) Reset() { *x = NodeGroupAutoscalingOptions{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptions) String() string { @@ -1774,7 +1687,7 @@ func (*NodeGroupAutoscalingOptions) ProtoMessage() {} func (x *NodeGroupAutoscalingOptions) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1824,24 +1737,35 @@ func (x *NodeGroupAutoscalingOptions) GetMaxNodeProvisionTime() *v1.Duration { return nil } -type NodeGroupAutoscalingOptionsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *NodeGroupAutoscalingOptions) GetZeroOrMaxNodeScaling() bool { + if x != nil { + return x.ZeroOrMaxNodeScaling + } + return false +} + +func (x *NodeGroupAutoscalingOptions) GetIgnoreDaemonSetsUtilization() bool { + if x != nil { + return x.IgnoreDaemonSetsUtilization + } + return false +} +type NodeGroupAutoscalingOptionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // default node group autoscaling options. - Defaults *NodeGroupAutoscalingOptions `protobuf:"bytes,2,opt,name=defaults,proto3" json:"defaults,omitempty"` + Defaults *NodeGroupAutoscalingOptions `protobuf:"bytes,2,opt,name=defaults,proto3" json:"defaults,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptionsRequest) Reset() { *x = NodeGroupAutoscalingOptionsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptionsRequest) String() string { @@ -1852,7 +1776,7 @@ func (*NodeGroupAutoscalingOptionsRequest) ProtoMessage() {} func (x *NodeGroupAutoscalingOptionsRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1882,21 +1806,18 @@ func (x *NodeGroupAutoscalingOptionsRequest) GetDefaults() *NodeGroupAutoscaling } type NodeGroupAutoscalingOptionsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // autoscaling options for the requested node. NodeGroupAutoscalingOptions *NodeGroupAutoscalingOptions `protobuf:"bytes,1,opt,name=nodeGroupAutoscalingOptions,proto3" json:"nodeGroupAutoscalingOptions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptionsResponse) Reset() { *x = NodeGroupAutoscalingOptionsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptionsResponse) String() string { @@ -1907,7 +1828,7 @@ func (*NodeGroupAutoscalingOptionsResponse) ProtoMessage() {} func (x *NodeGroupAutoscalingOptionsResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1931,463 +1852,148 @@ func (x *NodeGroupAutoscalingOptionsResponse) GetNodeGroupAutoscalingOptions() * var File_cloudprovider_externalgrpc_protos_externalgrpc_proto protoreflect.FileDescriptor -var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = []byte{ - 0x0a, 0x34, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x34, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, 0x6b, 0x38, 0x73, 0x2e, - 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x65, - 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6d, - 0x69, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x69, - 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x64, 0x65, 0x62, 0x75, 0x67, 0x22, 0x9e, 0x03, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x65, - 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, - 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x74, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x52, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x6e, - 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, - 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x13, 0x0a, 0x11, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x70, 0x0a, 0x12, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x5a, 0x0a, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x52, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x70, 0x0a, - 0x17, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, - 0x74, 0x0a, 0x18, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, - 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x6e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x09, 0x6e, 0x6f, 0x64, 0x65, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, - 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, - 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, - 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, - 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x30, 0x0a, 0x18, 0x50, 0x72, 0x69, 0x63, - 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x22, 0xd3, 0x01, 0x0a, 0x16, 0x50, - 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x03, 0x70, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x03, 0x70, 0x6f, 0x64, - 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, - 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, - 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x22, 0x2f, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, - 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, - 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x28, 0x0a, 0x10, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x1d, - 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, - 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xea, 0x01, - 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, - 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, - 0x0a, 0x08, 0x67, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x5b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, - 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x47, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x67, - 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x47, 0x70, 0x75, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x10, 0x0a, 0x0e, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x11, 0x0a, 0x0f, - 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x10, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x22, 0x3d, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, - 0x65, 0x22, 0x44, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, - 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1f, 0x0a, 0x1d, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, - 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x57, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x22, 0x1e, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x4a, 0x0a, 0x22, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, - 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x25, 0x0a, - 0x23, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, - 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x71, 0x0a, - 0x16, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, - 0x22, 0x73, 0x0a, 0x08, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x57, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xca, 0x02, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x73, 0x0a, 0x0d, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x60, 0x0a, - 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x22, - 0x61, 0x0a, 0x0d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x0f, 0x0a, 0x0b, 0x75, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x10, - 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x75, 0x6e, - 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6e, 0x67, - 0x10, 0x03, 0x22, 0x85, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x22, 0x32, 0x0a, 0x20, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, - 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x59, - 0x0a, 0x21, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, - 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xdd, 0x03, 0x0a, 0x1b, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, - 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x1d, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, - 0x52, 0x1d, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, - 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, - 0x4a, 0x0a, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, - 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x64, 0x0a, 0x15, 0x73, - 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6b, 0x38, 0x73, - 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, - 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x15, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x62, 0x0a, 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2e, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, - 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, - 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x62, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x4e, 0x6f, 0x64, 0x65, - 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x4d, 0x61, 0x78, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x9e, 0x01, 0x0a, 0x22, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, - 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x68, 0x0a, 0x08, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, - 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x08, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x23, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x8e, 0x01, 0x0a, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x32, 0xbf, 0x14, 0x0a, 0x0d, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x50, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x97, 0x01, 0x0a, 0x0a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x12, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x43, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0xa9, 0x01, 0x0a, 0x10, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, - 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa9, 0x01, 0x0a, 0x10, - 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, - 0x12, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, - 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa6, 0x01, 0x0a, 0x0f, 0x50, 0x72, 0x69, 0x63, - 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x47, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, - 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, - 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x91, 0x01, 0x0a, 0x08, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x40, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0xb5, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x4c, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, - 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, - 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, 0x0a, - 0x07, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, - 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, - 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, - 0x0a, 0x07, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, 0x72, - 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, - 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb2, - 0x01, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, - 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0xb8, 0x01, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4d, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4e, 0x2e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, - 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb5, - 0x01, 0x0a, 0x14, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xca, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0xa3, 0x01, 0x0a, 0x0e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x46, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x47, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xc4, 0x01, 0x0a, 0x19, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, - 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x51, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x52, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, - 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0xc2, 0x01, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x36, 0x5a, 0x34, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x2d, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2f, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = "" + + "\n" + + "4cloudprovider/externalgrpc/protos/externalgrpc.proto\x12/clusterautoscaler.cloudprovider.v1.externalgrpc\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\"k8s.io/api/core/v1/generated.proto\x1a4k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto\"e\n" + + "\tNodeGroup\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + + "\aminSize\x18\x02 \x01(\x05R\aminSize\x12\x18\n" + + "\amaxSize\x18\x03 \x01(\x05R\amaxSize\x12\x14\n" + + "\x05debug\x18\x04 \x01(\tR\x05debug\"\x9e\x03\n" + + "\x10ExternalGrpcNode\x12\x1e\n" + + "\n" + + "providerID\x18\x01 \x01(\tR\n" + + "providerID\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12e\n" + + "\x06labels\x18\x03 \x03(\v2M.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.LabelsEntryR\x06labels\x12t\n" + + "\vannotations\x18\x04 \x03(\v2R.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.AnnotationsEntryR\vannotations\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a>\n" + + "\x10AnnotationsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x13\n" + + "\x11NodeGroupsRequest\"p\n" + + "\x12NodeGroupsResponse\x12Z\n" + + "\n" + + "nodeGroups\x18\x01 \x03(\v2:.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupR\n" + + "nodeGroups\"p\n" + + "\x17NodeGroupForNodeRequest\x12U\n" + + "\x04node\x18\x01 \x01(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x04node\"t\n" + + "\x18NodeGroupForNodeResponse\x12X\n" + + "\tnodeGroup\x18\x01 \x01(\v2:.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupR\tnodeGroup\"\x80\x02\n" + + "\x17PricingNodePriceRequest\x12U\n" + + "\x04node\x18\x01 \x01(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x04node\x12H\n" + + "\tstartTime\x18\x02 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\tstartTime\x12D\n" + + "\aendTime\x18\x03 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\aendTime\"0\n" + + "\x18PricingNodePriceResponse\x12\x14\n" + + "\x05price\x18\x01 \x01(\x01R\x05price\"\xd3\x01\n" + + "\x16PricingPodPriceRequest\x12)\n" + + "\x03pod\x18\x01 \x01(\v2\x17.k8s.io.api.core.v1.PodR\x03pod\x12H\n" + + "\tstartTime\x18\x02 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\tstartTime\x12D\n" + + "\aendTime\x18\x03 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\aendTime\"/\n" + + "\x17PricingPodPriceResponse\x12\x14\n" + + "\x05price\x18\x01 \x01(\x01R\x05price\"\x11\n" + + "\x0fGPULabelRequest\"(\n" + + "\x10GPULabelResponse\x12\x14\n" + + "\x05label\x18\x01 \x01(\tR\x05label\"\x1d\n" + + "\x1bGetAvailableGPUTypesRequest\"\xea\x01\n" + + "\x1cGetAvailableGPUTypesResponse\x12w\n" + + "\bgpuTypes\x18\x01 \x03(\v2[.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.GpuTypesEntryR\bgpuTypes\x1aQ\n" + + "\rGpuTypesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12*\n" + + "\x05value\x18\x02 \x01(\v2\x14.google.protobuf.AnyR\x05value:\x028\x01\"\x10\n" + + "\x0eCleanupRequest\"\x11\n" + + "\x0fCleanupResponse\"\x10\n" + + "\x0eRefreshRequest\"\x11\n" + + "\x0fRefreshResponse\",\n" + + "\x1aNodeGroupTargetSizeRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"=\n" + + "\x1bNodeGroupTargetSizeResponse\x12\x1e\n" + + "\n" + + "targetSize\x18\x01 \x01(\x05R\n" + + "targetSize\"D\n" + + "\x1cNodeGroupIncreaseSizeRequest\x12\x14\n" + + "\x05delta\x18\x01 \x01(\x05R\x05delta\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"\x1f\n" + + "\x1dNodeGroupIncreaseSizeResponse\"\x86\x01\n" + + "\x1bNodeGroupDeleteNodesRequest\x12W\n" + + "\x05nodes\x18\x01 \x03(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x05nodes\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"\x1e\n" + + "\x1cNodeGroupDeleteNodesResponse\"J\n" + + "\"NodeGroupDecreaseTargetSizeRequest\x12\x14\n" + + "\x05delta\x18\x01 \x01(\x05R\x05delta\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"%\n" + + "#NodeGroupDecreaseTargetSizeResponse\"'\n" + + "\x15NodeGroupNodesRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"q\n" + + "\x16NodeGroupNodesResponse\x12W\n" + + "\tinstances\x18\x01 \x03(\v29.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceR\tinstances\"s\n" + + "\bInstance\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12W\n" + + "\x06status\x18\x02 \x01(\v2?.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatusR\x06status\"\xca\x02\n" + + "\x0eInstanceStatus\x12s\n" + + "\rinstanceState\x18\x01 \x01(\x0e2M.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceStateR\rinstanceState\x12`\n" + + "\terrorInfo\x18\x02 \x01(\v2B.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceErrorInfoR\terrorInfo\"a\n" + + "\rInstanceState\x12\x0f\n" + + "\vunspecified\x10\x00\x12\x13\n" + + "\x0finstanceRunning\x10\x01\x12\x14\n" + + "\x10instanceCreating\x10\x02\x12\x14\n" + + "\x10instanceDeleting\x10\x03\"\x85\x01\n" + + "\x11InstanceErrorInfo\x12\x1c\n" + + "\terrorCode\x18\x01 \x01(\tR\terrorCode\x12\"\n" + + "\ferrorMessage\x18\x02 \x01(\tR\ferrorMessage\x12.\n" + + "\x12instanceErrorClass\x18\x03 \x01(\x05R\x12instanceErrorClass\"2\n" + + " NodeGroupTemplateNodeInfoRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"Y\n" + + "!NodeGroupTemplateNodeInfoResponse\x124\n" + + "\bnodeInfo\x18\x01 \x01(\v2\x18.k8s.io.api.core.v1.NodeR\bnodeInfo\"\xd3\x04\n" + + "\x1bNodeGroupAutoscalingOptions\x12D\n" + + "\x1dscaleDownUtilizationThreshold\x18\x01 \x01(\x01R\x1dscaleDownUtilizationThreshold\x12J\n" + + " scaleDownGpuUtilizationThreshold\x18\x02 \x01(\x01R scaleDownGpuUtilizationThreshold\x12d\n" + + "\x15scaleDownUnneededTime\x18\x03 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x15scaleDownUnneededTime\x12b\n" + + "\x14scaleDownUnreadyTime\x18\x04 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x14scaleDownUnreadyTime\x12b\n" + + "\x14MaxNodeProvisionTime\x18\x05 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x14MaxNodeProvisionTime\x122\n" + + "\x14zeroOrMaxNodeScaling\x18\x06 \x01(\bR\x14zeroOrMaxNodeScaling\x12@\n" + + "\x1bignoreDaemonSetsUtilization\x18\a \x01(\bR\x1bignoreDaemonSetsUtilization\"\x9e\x01\n" + + "\"NodeGroupAutoscalingOptionsRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12h\n" + + "\bdefaults\x18\x02 \x01(\v2L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsR\bdefaults\"\xb6\x01\n" + + "#NodeGroupAutoscalingOptionsResponse\x12\x8e\x01\n" + + "\x1bnodeGroupAutoscalingOptions\x18\x01 \x01(\v2L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsR\x1bnodeGroupAutoscalingOptions2\xbf\x14\n" + + "\rCloudProvider\x12\x97\x01\n" + + "\n" + + "NodeGroups\x12B.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsRequest\x1aC.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsResponse\"\x00\x12\xa9\x01\n" + + "\x10NodeGroupForNode\x12H.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeRequest\x1aI.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeResponse\"\x00\x12\xa9\x01\n" + + "\x10PricingNodePrice\x12H.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest\x1aI.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceResponse\"\x00\x12\xa6\x01\n" + + "\x0fPricingPodPrice\x12G.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest\x1aH.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceResponse\"\x00\x12\x91\x01\n" + + "\bGPULabel\x12@.clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelRequest\x1aA.clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelResponse\"\x00\x12\xb5\x01\n" + + "\x14GetAvailableGPUTypes\x12L.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesRequest\x1aM.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse\"\x00\x12\x8e\x01\n" + + "\aCleanup\x12?.clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupRequest\x1a@.clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupResponse\"\x00\x12\x8e\x01\n" + + "\aRefresh\x12?.clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshRequest\x1a@.clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshResponse\"\x00\x12\xb2\x01\n" + + "\x13NodeGroupTargetSize\x12K.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeRequest\x1aL.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeResponse\"\x00\x12\xb8\x01\n" + + "\x15NodeGroupIncreaseSize\x12M.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeRequest\x1aN.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeResponse\"\x00\x12\xb5\x01\n" + + "\x14NodeGroupDeleteNodes\x12L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesRequest\x1aM.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesResponse\"\x00\x12\xca\x01\n" + + "\x1bNodeGroupDecreaseTargetSize\x12S.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeRequest\x1aT.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeResponse\"\x00\x12\xa3\x01\n" + + "\x0eNodeGroupNodes\x12F.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesRequest\x1aG.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesResponse\"\x00\x12\xc4\x01\n" + + "\x19NodeGroupTemplateNodeInfo\x12Q.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoRequest\x1aR.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoResponse\"\x00\x12\xc2\x01\n" + + "\x13NodeGroupGetOptions\x12S.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsRequest\x1aT.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsResponse\"\x00B6Z4cluster-autoscaler/cloudprovider/externalgrpc/protosb\x06proto3" var ( file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce sync.Once - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData []byte ) func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP() []byte { file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce.Do(func() { - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData) + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc), len(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc))) }) return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData } var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes = make([]protoimpl.MessageInfo, 39) -var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = []interface{}{ +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = []any{ (InstanceStatus_InstanceState)(0), // 0: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceState (*NodeGroup)(nil), // 1: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroup (*ExternalGrpcNode)(nil), // 2: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode @@ -2501,445 +2107,11 @@ func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() { if File_cloudprovider_externalgrpc_protos_externalgrpc_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroup); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExternalGrpcNode); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupForNodeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupForNodeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingNodePriceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingNodePriceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingPodPriceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingPodPriceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GPULabelRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GPULabelResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetAvailableGPUTypesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetAvailableGPUTypesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanupRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanupResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefreshRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefreshResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTargetSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTargetSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupIncreaseSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupIncreaseSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDeleteNodesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDeleteNodesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDecreaseTargetSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDecreaseTargetSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupNodesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupNodesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Instance); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstanceStatus); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstanceErrorInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTemplateNodeInfoRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTemplateNodeInfoResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptions); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptionsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptionsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc), len(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc)), NumEnums: 1, NumMessages: 39, NumExtensions: 0, @@ -2951,7 +2123,6 @@ func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() { MessageInfos: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes, }.Build() File_cloudprovider_externalgrpc_protos_externalgrpc_proto = out.File - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = nil file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = nil file_cloudprovider_externalgrpc_protos_externalgrpc_proto_depIdxs = nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto index 0a476e65973..0e87886666d 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto @@ -1,28 +1,27 @@ /* -Copyright 2022 The Kubernetes Authors. + Copyright 2022 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ syntax = "proto3"; package clusterautoscaler.cloudprovider.v1.externalgrpc; -import "google/protobuf/descriptor.proto"; import "google/protobuf/any.proto"; - -import "k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto"; +import "google/protobuf/descriptor.proto"; import "k8s.io/api/core/v1/generated.proto"; +import "k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto"; option go_package = "cluster-autoscaler/cloudprovider/externalgrpc/protos"; @@ -30,42 +29,34 @@ service CloudProvider { // CloudProvider specific RPC functions // NodeGroups returns all node groups configured for this cloud provider. - rpc NodeGroups(NodeGroupsRequest) - returns (NodeGroupsResponse) {} + rpc NodeGroups(NodeGroupsRequest) returns (NodeGroupsResponse) {} // NodeGroupForNode returns the node group for the given node. // The node group id is an empty string if the node should not // be processed by cluster autoscaler. - rpc NodeGroupForNode(NodeGroupForNodeRequest) - returns (NodeGroupForNodeResponse) {} + rpc NodeGroupForNode(NodeGroupForNodeRequest) returns (NodeGroupForNodeResponse) {} // PricingNodePrice returns a theoretical minimum price of running a node for // a given period of time on a perfectly matching machine. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc PricingNodePrice(PricingNodePriceRequest) - returns (PricingNodePriceResponse) {} + rpc PricingNodePrice(PricingNodePriceRequest) returns (PricingNodePriceResponse) {} // PricingPodPrice returns a theoretical minimum price of running a pod for a given // period of time on a perfectly matching machine. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc PricingPodPrice(PricingPodPriceRequest) - returns (PricingPodPriceResponse) {} + rpc PricingPodPrice(PricingPodPriceRequest) returns (PricingPodPriceResponse) {} // GPULabel returns the label added to nodes with GPU resource. - rpc GPULabel(GPULabelRequest) - returns (GPULabelResponse) {} + rpc GPULabel(GPULabelRequest) returns (GPULabelResponse) {} // GetAvailableGPUTypes return all available GPU types cloud provider supports. - rpc GetAvailableGPUTypes(GetAvailableGPUTypesRequest) - returns (GetAvailableGPUTypesResponse) {} + rpc GetAvailableGPUTypes(GetAvailableGPUTypesRequest) returns (GetAvailableGPUTypesResponse) {} // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. - rpc Cleanup(CleanupRequest) - returns (CleanupResponse) {} + rpc Cleanup(CleanupRequest) returns (CleanupResponse) {} // Refresh is called before every main loop and can be used to dynamically update cloud provider state. - rpc Refresh(RefreshRequest) - returns (RefreshResponse) {} + rpc Refresh(RefreshRequest) returns (RefreshResponse) {} // NodeGroup specific RPC functions @@ -73,45 +64,38 @@ service CloudProvider { // that the number of nodes in Kubernetes is different at the moment but should be equal // to the size of a node group once everything stabilizes (new nodes finish startup and // registration or removed nodes are deleted completely). - rpc NodeGroupTargetSize(NodeGroupTargetSizeRequest) - returns (NodeGroupTargetSizeResponse) {} + rpc NodeGroupTargetSize(NodeGroupTargetSizeRequest) returns (NodeGroupTargetSizeResponse) {} // NodeGroupIncreaseSize increases the size of the node group. To delete a node you need // to explicitly name it and use NodeGroupDeleteNodes. This function should wait until // node group size is updated. - rpc NodeGroupIncreaseSize(NodeGroupIncreaseSizeRequest) - returns (NodeGroupIncreaseSizeResponse) {} + rpc NodeGroupIncreaseSize(NodeGroupIncreaseSizeRequest) returns (NodeGroupIncreaseSizeResponse) {} // NodeGroupDeleteNodes deletes nodes from this node group (and also decreasing the size // of the node group with that). Error is returned either on failure or if the given node // doesn't belong to this node group. This function should wait until node group size is updated. - rpc NodeGroupDeleteNodes(NodeGroupDeleteNodesRequest) - returns (NodeGroupDeleteNodesResponse) {} + rpc NodeGroupDeleteNodes(NodeGroupDeleteNodesRequest) returns (NodeGroupDeleteNodesResponse) {} // NodeGroupDecreaseTargetSize decreases the target size of the node group. This function // doesn't permit to delete any existing node and can be used only to reduce the request // for new nodes that have not been yet fulfilled. Delta should be negative. It is assumed // that cloud provider will not delete the existing nodes if the size when there is an option // to just decrease the target. - rpc NodeGroupDecreaseTargetSize(NodeGroupDecreaseTargetSizeRequest) - returns (NodeGroupDecreaseTargetSizeResponse) {} + rpc NodeGroupDecreaseTargetSize(NodeGroupDecreaseTargetSizeRequest) returns (NodeGroupDecreaseTargetSizeResponse) {} // NodeGroupNodes returns a list of all nodes that belong to this node group. - rpc NodeGroupNodes(NodeGroupNodesRequest) - returns (NodeGroupNodesResponse) {} + rpc NodeGroupNodes(NodeGroupNodesRequest) returns (NodeGroupNodesResponse) {} // NodeGroupTemplateNodeInfo returns a structure of an empty (as if just started) node, // with all of the labels, capacity and allocatable information. This will be used in // scale-up simulations to predict what would a new node look like if a node group was expanded. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc NodeGroupTemplateNodeInfo(NodeGroupTemplateNodeInfoRequest) - returns (NodeGroupTemplateNodeInfoResponse) {} + rpc NodeGroupTemplateNodeInfo(NodeGroupTemplateNodeInfoRequest) returns (NodeGroupTemplateNodeInfoResponse) {} // GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular // NodeGroup. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc NodeGroupGetOptions(NodeGroupAutoscalingOptionsRequest) - returns (NodeGroupAutoscalingOptionsResponse) {} + rpc NodeGroupGetOptions(NodeGroupAutoscalingOptionsRequest) returns (NodeGroupAutoscalingOptionsResponse) {} } message NodeGroup { @@ -128,7 +112,7 @@ message NodeGroup { string debug = 4; } -message ExternalGrpcNode{ +message ExternalGrpcNode { // ID of the node assigned by the cloud provider in the format: ://. string providerID = 1; @@ -259,7 +243,7 @@ message NodeGroupDeleteNodesRequest { message NodeGroupDeleteNodesResponse { // Intentionally empty. - } +} message NodeGroupDecreaseTargetSizeRequest { // Number of nodes to delete. @@ -343,7 +327,7 @@ message NodeGroupAutoscalingOptions { // if cpu or memory utilization is over threshold. double scaleDownUtilizationThreshold = 1; - // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be + // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be // considered for scale down if gpu utilization is over threshold. double scaleDownGpuUtilizationThreshold = 2; @@ -356,7 +340,13 @@ message NodeGroupAutoscalingOptions { k8s.io.apimachinery.pkg.apis.meta.v1.Duration scaleDownUnreadyTime = 4; // MaxNodeProvisionTime time CA waits for node to be provisioned - k8s.io.apimachinery.pkg.apis.meta.v1.Duration MaxNodeProvisionTime = 5; + k8s.io.apimachinery.pkg.apis.meta.v1.Duration MaxNodeProvisionTime = 5; + + // ZeroOrMaxNodeScaling means that a node group should be scaled up to maximum size or down to zero nodes all at once instead of one-by-one. + bool zeroOrMaxNodeScaling = 6; + + // IgnoreDaemonSetsUtilization sets if daemonsets utilization should be considered during node scale-down + bool ignoreDaemonSetsUtilization = 7; } message NodeGroupAutoscalingOptionsRequest { diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go index 1b2b4f38a37..e788f5d071e 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go @@ -15,8 +15,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.5.1 +// - protoc v5.29.2 // source: cloudprovider/externalgrpc/protos/externalgrpc.proto package protos @@ -30,8 +30,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 const ( CloudProvider_NodeGroups_FullMethodName = "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroups" @@ -118,8 +118,9 @@ func NewCloudProviderClient(cc grpc.ClientConnInterface) CloudProviderClient { } func (c *cloudProviderClient) NodeGroups(ctx context.Context, in *NodeGroupsRequest, opts ...grpc.CallOption) (*NodeGroupsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupsResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroups_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroups_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -127,8 +128,9 @@ func (c *cloudProviderClient) NodeGroups(ctx context.Context, in *NodeGroupsRequ } func (c *cloudProviderClient) NodeGroupForNode(ctx context.Context, in *NodeGroupForNodeRequest, opts ...grpc.CallOption) (*NodeGroupForNodeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupForNodeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupForNode_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupForNode_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -136,8 +138,9 @@ func (c *cloudProviderClient) NodeGroupForNode(ctx context.Context, in *NodeGrou } func (c *cloudProviderClient) PricingNodePrice(ctx context.Context, in *PricingNodePriceRequest, opts ...grpc.CallOption) (*PricingNodePriceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PricingNodePriceResponse) - err := c.cc.Invoke(ctx, CloudProvider_PricingNodePrice_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_PricingNodePrice_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -145,8 +148,9 @@ func (c *cloudProviderClient) PricingNodePrice(ctx context.Context, in *PricingN } func (c *cloudProviderClient) PricingPodPrice(ctx context.Context, in *PricingPodPriceRequest, opts ...grpc.CallOption) (*PricingPodPriceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PricingPodPriceResponse) - err := c.cc.Invoke(ctx, CloudProvider_PricingPodPrice_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_PricingPodPrice_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -154,8 +158,9 @@ func (c *cloudProviderClient) PricingPodPrice(ctx context.Context, in *PricingPo } func (c *cloudProviderClient) GPULabel(ctx context.Context, in *GPULabelRequest, opts ...grpc.CallOption) (*GPULabelResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GPULabelResponse) - err := c.cc.Invoke(ctx, CloudProvider_GPULabel_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_GPULabel_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -163,8 +168,9 @@ func (c *cloudProviderClient) GPULabel(ctx context.Context, in *GPULabelRequest, } func (c *cloudProviderClient) GetAvailableGPUTypes(ctx context.Context, in *GetAvailableGPUTypesRequest, opts ...grpc.CallOption) (*GetAvailableGPUTypesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetAvailableGPUTypesResponse) - err := c.cc.Invoke(ctx, CloudProvider_GetAvailableGPUTypes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_GetAvailableGPUTypes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -172,8 +178,9 @@ func (c *cloudProviderClient) GetAvailableGPUTypes(ctx context.Context, in *GetA } func (c *cloudProviderClient) Cleanup(ctx context.Context, in *CleanupRequest, opts ...grpc.CallOption) (*CleanupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CleanupResponse) - err := c.cc.Invoke(ctx, CloudProvider_Cleanup_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_Cleanup_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -181,8 +188,9 @@ func (c *cloudProviderClient) Cleanup(ctx context.Context, in *CleanupRequest, o } func (c *cloudProviderClient) Refresh(ctx context.Context, in *RefreshRequest, opts ...grpc.CallOption) (*RefreshResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RefreshResponse) - err := c.cc.Invoke(ctx, CloudProvider_Refresh_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_Refresh_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -190,8 +198,9 @@ func (c *cloudProviderClient) Refresh(ctx context.Context, in *RefreshRequest, o } func (c *cloudProviderClient) NodeGroupTargetSize(ctx context.Context, in *NodeGroupTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupTargetSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupTargetSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTargetSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTargetSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -199,8 +208,9 @@ func (c *cloudProviderClient) NodeGroupTargetSize(ctx context.Context, in *NodeG } func (c *cloudProviderClient) NodeGroupIncreaseSize(ctx context.Context, in *NodeGroupIncreaseSizeRequest, opts ...grpc.CallOption) (*NodeGroupIncreaseSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupIncreaseSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupIncreaseSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupIncreaseSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -208,8 +218,9 @@ func (c *cloudProviderClient) NodeGroupIncreaseSize(ctx context.Context, in *Nod } func (c *cloudProviderClient) NodeGroupDeleteNodes(ctx context.Context, in *NodeGroupDeleteNodesRequest, opts ...grpc.CallOption) (*NodeGroupDeleteNodesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupDeleteNodesResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDeleteNodes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDeleteNodes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -217,8 +228,9 @@ func (c *cloudProviderClient) NodeGroupDeleteNodes(ctx context.Context, in *Node } func (c *cloudProviderClient) NodeGroupDecreaseTargetSize(ctx context.Context, in *NodeGroupDecreaseTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupDecreaseTargetSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupDecreaseTargetSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDecreaseTargetSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDecreaseTargetSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -226,8 +238,9 @@ func (c *cloudProviderClient) NodeGroupDecreaseTargetSize(ctx context.Context, i } func (c *cloudProviderClient) NodeGroupNodes(ctx context.Context, in *NodeGroupNodesRequest, opts ...grpc.CallOption) (*NodeGroupNodesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupNodesResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupNodes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupNodes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -235,8 +248,9 @@ func (c *cloudProviderClient) NodeGroupNodes(ctx context.Context, in *NodeGroupN } func (c *cloudProviderClient) NodeGroupTemplateNodeInfo(ctx context.Context, in *NodeGroupTemplateNodeInfoRequest, opts ...grpc.CallOption) (*NodeGroupTemplateNodeInfoResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupTemplateNodeInfoResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTemplateNodeInfo_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTemplateNodeInfo_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -244,8 +258,9 @@ func (c *cloudProviderClient) NodeGroupTemplateNodeInfo(ctx context.Context, in } func (c *cloudProviderClient) NodeGroupGetOptions(ctx context.Context, in *NodeGroupAutoscalingOptionsRequest, opts ...grpc.CallOption) (*NodeGroupAutoscalingOptionsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupAutoscalingOptionsResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupGetOptions_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupGetOptions_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -254,7 +269,7 @@ func (c *cloudProviderClient) NodeGroupGetOptions(ctx context.Context, in *NodeG // CloudProviderServer is the server API for CloudProvider service. // All implementations must embed UnimplementedCloudProviderServer -// for forward compatibility +// for forward compatibility. type CloudProviderServer interface { // NodeGroups returns all node groups configured for this cloud provider. NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) @@ -311,9 +326,12 @@ type CloudProviderServer interface { mustEmbedUnimplementedCloudProviderServer() } -// UnimplementedCloudProviderServer must be embedded to have forward compatible implementations. -type UnimplementedCloudProviderServer struct { -} +// UnimplementedCloudProviderServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedCloudProviderServer struct{} func (UnimplementedCloudProviderServer) NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method NodeGroups not implemented") @@ -361,6 +379,7 @@ func (UnimplementedCloudProviderServer) NodeGroupGetOptions(context.Context, *No return nil, status.Errorf(codes.Unimplemented, "method NodeGroupGetOptions not implemented") } func (UnimplementedCloudProviderServer) mustEmbedUnimplementedCloudProviderServer() {} +func (UnimplementedCloudProviderServer) testEmbeddedByValue() {} // UnsafeCloudProviderServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to CloudProviderServer will @@ -370,6 +389,13 @@ type UnsafeCloudProviderServer interface { } func RegisterCloudProviderServer(s grpc.ServiceRegistrar, srv CloudProviderServer) { + // If the following call pancis, it indicates UnimplementedCloudProviderServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } s.RegisterService(&CloudProvider_ServiceDesc, srv) } From 63ed537f70beb5eed395f2bc2535fbb20a7cf686 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Tue, 1 Apr 2025 06:55:18 +0000 Subject: [PATCH 33/39] Add docs for in-place updates Signed-off-by: Omer Aplatony --- vertical-pod-autoscaler/docs/features.md | 75 +++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index bc1f8d90f48..668864b2695 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -4,6 +4,8 @@ - [Limits control](#limits-control) - [Memory Value Humanization](#memory-value-humanization) +- [CPU Recommendation Rounding](#cpu-recommendation-rounding) +- [In-Place Updates](#in-place-updates) ## Limits control @@ -50,4 +52,75 @@ To enable this feature, set the --round-cpu-millicores flag when running the VPA ```bash --round-cpu-millicores=50 -``` \ No newline at end of file +``` + +## In-Place Updates + +VPA supports in-place updates to reduce disruption when applying resource recommendations. This feature leverages Kubernetes' in-place update capabilities (beta in 1.33) to modify container resources without requiring pod recreation. + +### Usage + +To use in-place updates, set the VPA's `updateMode` to `InPlaceOrRecreate`: +```yaml +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: my-vpa +spec: + updatePolicy: + updateMode: "InPlaceOrRecreate" +``` + +### Behavior + +When using `InPlaceOrRecreate` mode, VPA will first attempt to apply updates in-place, if in-place update fails, VPA will fall back to pod recreation. +Updates are attempted when: +* Container requests are outside the recommended bounds +* Quick OOM occurs +* For long-running pods (>12h), when recommendations differ significantly (>10%) + +Important Notes + +* Disruption Possibility: While in-place updates aim to minimize disruption, they cannot guarantee zero disruption as the container runtime is responsible for the actual resize operation. + +* Memory Limit Downscaling: In the beta version, memory limit downscaling is not supported for pods with resizePolicy: PreferNoRestart. In such cases, VPA will fall back to pod recreation. + +### Requirements: + +* Kubernetes 1.33+ with in-place update feature enabled +* VPA version 1.4.0+ with `InPlaceOrRecreate` feature gate enabled +* Cluster feature gate InPlacePodVerticalScaling enabled + +### Configuration + +Enable the feature by setting the following flags in VPA components ( for both updater and admission-controller ): + +```bash +--feature-gates=InPlaceOrRecreate=true +``` + +### Limitations + +* All containers in a pod are updated together (partial updates not supported) +* Memory downscaling requires careful consideration to prevent OOMs +* Updates still respect VPA's standard update conditions and timing restrictions +* In-place updates will fail for pods with Guaranteed QoS class (requires pod recreation) + +### Fallback Behavior + +VPA will fall back to pod recreation in the following scenarios: + +* In-place update is [infeasible](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#resize-status) (node resources, etc.) +* Update is [deferred](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#resize-status) for more than 5 minutes +* Update is in progress for more than 1 hour +* [Pod QoS](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/) class would change due to the update +* Memory limit downscaling is required with [PreferNoRestart policy](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#container-resize-policy) + +### Monitoring + +VPA provides metrics to track in-place update operations: + +* vpa_in_place_updatable_pods_total: Number of pods matching in-place update criteria +* vpa_in_place_updated_pods_total: Number of pods successfully updated in-place +* vpa_vpas_with_in_place_updatable_pods_total: Number of VPAs with pods eligible for in-place updates +* vpa_vpas_with_in_place_updated_pods_total: Number of VPAs with successfully in-place updated pods \ No newline at end of file From 50a1035c042113eab63b8a6212471a42c6b49ed1 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Wed, 2 Apr 2025 06:46:58 +0000 Subject: [PATCH 34/39] Add AEP link and refine intro for VPA in-place updates documentation Signed-off-by: Omer Aplatony --- vertical-pod-autoscaler/docs/features.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index 668864b2695..44e0d16dc03 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -57,6 +57,7 @@ To enable this feature, set the --round-cpu-millicores flag when running the VPA ## In-Place Updates VPA supports in-place updates to reduce disruption when applying resource recommendations. This feature leverages Kubernetes' in-place update capabilities (beta in 1.33) to modify container resources without requiring pod recreation. +For more information, see [AEP-4916: Support for in place updates in VPA](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support) ### Usage @@ -87,9 +88,9 @@ Important Notes ### Requirements: -* Kubernetes 1.33+ with in-place update feature enabled +* Kubernetes 1.33+ * VPA version 1.4.0+ with `InPlaceOrRecreate` feature gate enabled -* Cluster feature gate InPlacePodVerticalScaling enabled +* Cluster feature gate `InPlacePodVerticalScaling` enabled ### Configuration @@ -120,7 +121,7 @@ VPA will fall back to pod recreation in the following scenarios: VPA provides metrics to track in-place update operations: -* vpa_in_place_updatable_pods_total: Number of pods matching in-place update criteria -* vpa_in_place_updated_pods_total: Number of pods successfully updated in-place -* vpa_vpas_with_in_place_updatable_pods_total: Number of VPAs with pods eligible for in-place updates -* vpa_vpas_with_in_place_updated_pods_total: Number of VPAs with successfully in-place updated pods \ No newline at end of file +* `vpa_in_place_updatable_pods_total`: Number of pods matching in-place update criteria +* `vpa_in_place_updated_pods_total`: Number of pods successfully updated in-place +* `vpa_vpas_with_in_place_updatable_pods_total`: Number of VPAs with pods eligible for in-place updates +* `vpa_vpas_with_in_place_updated_pods_total`: Number of VPAs with successfully in-place updated pods \ No newline at end of file From a34a3527d91ec01ff8dcb74a52ebaa520df533b9 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Wed, 2 Apr 2025 06:50:43 +0000 Subject: [PATCH 35/39] add feature state Signed-off-by: Omer Aplatony --- vertical-pod-autoscaler/docs/features.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index 44e0d16dc03..8e043a699df 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -54,7 +54,10 @@ To enable this feature, set the --round-cpu-millicores flag when running the VPA --round-cpu-millicores=50 ``` -## In-Place Updates +## In-Place Updates (`InPlaceOrRecreate`) + +> [!NOTE] +> FEATURE STATE: VPA v1.4.0 [alpha] VPA supports in-place updates to reduce disruption when applying resource recommendations. This feature leverages Kubernetes' in-place update capabilities (beta in 1.33) to modify container resources without requiring pod recreation. For more information, see [AEP-4916: Support for in place updates in VPA](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support) From e3fbeffae993864c1b85d103904d091b19546c17 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Wed, 2 Apr 2025 20:38:21 +0300 Subject: [PATCH 36/39] Update vertical-pod-autoscaler/docs/features.md Co-authored-by: Adrian Moisey --- vertical-pod-autoscaler/docs/features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index 8e043a699df..e67d5f613a0 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -5,7 +5,7 @@ - [Limits control](#limits-control) - [Memory Value Humanization](#memory-value-humanization) - [CPU Recommendation Rounding](#cpu-recommendation-rounding) -- [In-Place Updates](#in-place-updates) +- [In-Place Updates](#in-place-updates-inplaceorrecreate) ## Limits control From 3f2845c2da70cc7f4f58c688409163435c147644 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Wed, 2 Apr 2025 20:39:39 +0300 Subject: [PATCH 37/39] Update vertical-pod-autoscaler/docs/features.md Co-authored-by: Adrian Moisey --- vertical-pod-autoscaler/docs/features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index e67d5f613a0..e57911c49d4 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -56,7 +56,7 @@ To enable this feature, set the --round-cpu-millicores flag when running the VPA ## In-Place Updates (`InPlaceOrRecreate`) -> [!NOTE] +> [!WARNING] > FEATURE STATE: VPA v1.4.0 [alpha] VPA supports in-place updates to reduce disruption when applying resource recommendations. This feature leverages Kubernetes' in-place update capabilities (beta in 1.33) to modify container resources without requiring pod recreation. From 5b5ae394056b85f437a35ca2708b0120de2fb62d Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Thu, 3 Apr 2025 15:24:53 +0300 Subject: [PATCH 38/39] Update vertical-pod-autoscaler/docs/features.md Co-authored-by: Adrian Moisey --- vertical-pod-autoscaler/docs/features.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index e57911c49d4..4c8635e38ff 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -91,9 +91,8 @@ Important Notes ### Requirements: -* Kubernetes 1.33+ +* Kubernetes 1.33+ with `InPlacePodVerticalScaling` feature gate enabled * VPA version 1.4.0+ with `InPlaceOrRecreate` feature gate enabled -* Cluster feature gate `InPlacePodVerticalScaling` enabled ### Configuration From 125209d9adecce276ad975e1028fd57064125083 Mon Sep 17 00:00:00 2001 From: Omer Aplatony Date: Sun, 6 Apr 2025 16:49:37 +0000 Subject: [PATCH 39/39] fixed wrong statement Signed-off-by: Omer Aplatony --- vertical-pod-autoscaler/docs/features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index 4c8635e38ff..1a15863936b 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -107,7 +107,7 @@ Enable the feature by setting the following flags in VPA components ( for both u * All containers in a pod are updated together (partial updates not supported) * Memory downscaling requires careful consideration to prevent OOMs * Updates still respect VPA's standard update conditions and timing restrictions -* In-place updates will fail for pods with Guaranteed QoS class (requires pod recreation) +* In-place updates will fail if they would result in a change to the pod's QoS class ### Fallback Behavior