diff --git a/cluster-autoscaler/cloudprovider/azure/azure_client.go b/cluster-autoscaler/cloudprovider/azure/azure_client.go index fbc39a62ed2..b875f32168d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_client.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_client.go @@ -30,7 +30,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/azure/auth" diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go index 967177cafa6..bf6cca92267 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller.go @@ -55,6 +55,7 @@ const ( resourceNameMachineSet = "machinesets" resourceNameMachineDeployment = "machinedeployments" resourceNameMachinePool = "machinepools" + deletingMachinePrefix = "deleting-machine-" failedMachinePrefix = "failed-machine-" pendingMachinePrefix = "pending-machine-" machineTemplateKind = "MachineTemplate" @@ -314,6 +315,9 @@ func (c *machineController) findMachineByProviderID(providerID normalizedProvide return u.DeepCopy(), nil } + if isDeletingMachineProviderID(providerID) { + return c.findMachine(machineKeyFromDeletingMachineProviderID(providerID)) + } if isFailedMachineProviderID(providerID) { return c.findMachine(machineKeyFromFailedProviderID(providerID)) } @@ -339,6 +343,24 @@ func (c *machineController) findMachineByProviderID(providerID normalizedProvide return c.findMachine(path.Join(ns, machineID)) } +func createDeletingMachineNormalizedProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", deletingMachinePrefix, namespace, name) +} + +func isDeletingMachineProviderID(providerID normalizedProviderID) bool { + return strings.HasPrefix(string(providerID), deletingMachinePrefix) +} + +func machineKeyFromDeletingMachineProviderID(providerID normalizedProviderID) string { + namespaceName := strings.TrimPrefix(string(providerID), deletingMachinePrefix) + return strings.Replace(namespaceName, "_", "/", 1) +} + +// createPendingMachineProviderID creates a providerID for a machine that is pending +func createPendingMachineProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", pendingMachinePrefix, namespace, name) +} + func isPendingMachineProviderID(providerID normalizedProviderID) bool { return strings.HasPrefix(string(providerID), pendingMachinePrefix) } @@ -348,6 +370,10 @@ func machineKeyFromPendingMachineProviderID(providerID normalizedProviderID) str return strings.Replace(namespaceName, "_", "/", 1) } +func createFailedMachineNormalizedProviderID(namespace, name string) string { + return fmt.Sprintf("%s%s_%s", failedMachinePrefix, namespace, name) +} + func isFailedMachineProviderID(providerID normalizedProviderID) bool { return strings.HasPrefix(string(providerID), failedMachinePrefix) } @@ -357,6 +383,15 @@ func machineKeyFromFailedProviderID(providerID normalizedProviderID) string { return strings.Replace(namespaceName, "_", "/", 1) } +// isProviderIDNormalized determines whether a node's providerID is the standard +// providerID assigned by the cloud provider, or if it has +// been modified by the CAS CAPI provider to indicate deleting, pending, or failed +func isProviderIDNormalized(providerID normalizedProviderID) bool { + return !isDeletingMachineProviderID(providerID) && + !isPendingMachineProviderID(providerID) && + !isFailedMachineProviderID(providerID) +} + // findNodeByNodeName finds the Node object keyed by name.. Returns // nil if it cannot be found. A DeepCopy() of the object is returned // on success. @@ -603,36 +638,46 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un } for _, machine := range machines { - providerID, found, err := unstructured.NestedString(machine.UnstructuredContent(), "spec", "providerID") - if err != nil { - return nil, err - } - - if found { - if providerID != "" { - providerIDs = append(providerIDs, providerID) - continue - } - } - - klog.Warningf("Machine %q has no providerID", machine.GetName()) - + // Failed Machines + // In some cases it is possible for a machine to have acquired a provider ID from the infrastructure and + // then become failed later. We want to ensure that a failed machine is not counted towards the total + // number of nodes in the cluster, for this reason we will detect a failed machine first, regardless + // of provider ID, and give it a normalized provider ID with failure message prepended. failureMessage, found, err := unstructured.NestedString(machine.UnstructuredContent(), "status", "failureMessage") if err != nil { return nil, err } if found { - klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) - // Provide a fake ID to allow the autoscaler to track machines that will never + // Provide a normalized ID to allow the autoscaler to track machines that will never // become nodes and mark the nodegroup unhealthy after maxNodeProvisionTime. // Fake ID needs to be recognised later and converted into a machine key. // Use an underscore as a separator between namespace and name as it is not a // valid character within a namespace name. - providerIDs = append(providerIDs, fmt.Sprintf("%s%s_%s", failedMachinePrefix, machine.GetNamespace(), machine.GetName())) + klog.V(4).Infof("Status.FailureMessage of machine %q is %q", machine.GetName(), failureMessage) + providerIDs = append(providerIDs, createFailedMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName())) continue } + // Deleting Machines + // Machines that are in deleting state should be identified so that in scenarios where the core + // autoscaler would like to adjust the size of a node group, we can give a proper count and + // be able to filter machines in that state, regardless of whether they are still active nodes in the cluster. + // We give these machines normalized provider IDs to aid in the filtering process. + if !machine.GetDeletionTimestamp().IsZero() { + klog.V(4).Infof("Machine %q has a non-zero deletion timestamp", machine.GetName()) + providerIDs = append(providerIDs, createDeletingMachineNormalizedProviderID(machine.GetNamespace(), machine.GetName())) + continue + } + + // Pending Machines + // Machines that do not yet have an associated node reference are considering to be pending. These + // nodes need to be filtered so that in a case where a machine is not becoming a node, or the instance + // lifecycle has changed during provisioning (eg spot instance going away), or the core autoscaler has + // decided that the node is not needed. + // Look for a node reference in the status, a machine without a node reference, and that is also not + // in failed or deleting state, has not yet become a node, and should be marked as pending. + // We give these machines normalized provider IDs to aid in the filtering process. _, found, err = unstructured.NestedFieldCopy(machine.UnstructuredContent(), "status", "nodeRef") if err != nil { return nil, err @@ -640,10 +685,33 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un if !found { klog.V(4).Infof("Status.NodeRef of machine %q is currently nil", machine.GetName()) - providerIDs = append(providerIDs, fmt.Sprintf("%s%s_%s", pendingMachinePrefix, machine.GetNamespace(), machine.GetName())) + providerIDs = append(providerIDs, createPendingMachineProviderID(machine.GetNamespace(), machine.GetName())) continue } + // Running Machines + // We have filtered out the machines in failed, deleting, and pending states. We now check the provider + // ID and potentially the node reference details. It is ok for a machine not to have a provider ID as + // not all CAPI provider implement this field, but a machine in running state should have a valid + // node reference. If a provider ID is present, we add that to the list as we know it is not failed, + // deleting, or pending. If an empty provider ID is present, we check the node details to ensure that + // the machine references a valid node. + providerID, found, err := unstructured.NestedString(machine.UnstructuredContent(), "spec", "providerID") + if err != nil { + return nil, err + } + + if found { + if providerID != "" { + // Machine has a provider ID, add it to the list + providerIDs = append(providerIDs, providerID) + continue + } + } + + klog.Warningf("Machine %q has no providerID", machine.GetName()) + + // Begin checking to determine if the node reference is valid nodeRefKind, found, err := unstructured.NestedString(machine.UnstructuredContent(), "status", "nodeRef", "kind") if err != nil { return nil, err @@ -665,6 +733,8 @@ func (c *machineController) findScalableResourceProviderIDs(scalableResource *un return nil, fmt.Errorf("unknown node %q", nodeRefName) } + // A node has been found that corresponds to this machine, since we know that this machine has + // an empty provider ID, we add the provider ID from the node to the list. if node != nil { providerIDs = append(providerIDs, node.Spec.ProviderID) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go index 4e63a7fa640..fadc07aa783 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_controller_test.go @@ -1335,19 +1335,6 @@ func TestControllerMachineSetNodeNamesUsingProviderID(t *testing.T) { controller, stop := mustCreateTestController(t, testConfig) defer stop() - // Remove Status.NodeRef.Name on all the machines. We want to - // force machineSetNodeNames() to only consider the provider - // ID for lookups. - for i := range testConfig.machines { - machine := testConfig.machines[i].DeepCopy() - - unstructured.RemoveNestedField(machine.Object, "status", "nodeRef") - - if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { - t.Fatalf("unexpected error updating machine, got %v", err) - } - } - nodegroups, err := controller.nodeGroups() if err != nil { t.Fatalf("unexpected error: %v", err) @@ -1664,6 +1651,11 @@ func TestIsFailedMachineProviderID(t *testing.T) { providerID: normalizedProviderID("foo"), expected: false, }, + { + name: "with provider ID created by createFailedMachineNormalizedProviderID should return true", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expected: true, + }, } for _, tc := range testCases { @@ -1691,6 +1683,11 @@ func TestMachineKeyFromFailedProviderID(t *testing.T) { providerID: normalizedProviderID(fmt.Sprintf("%stest-namespace_foo_bar", failedMachinePrefix)), expected: "test-namespace/foo_bar", }, + { + name: "with a provider ID created by createFailedMachineNormalizedProviderID", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expected: "cluster-api/id-0001", + }, } for _, tc := range testCases { @@ -2131,3 +2128,187 @@ func Test_machineController_nodeGroups(t *testing.T) { }) } } + +func Test_isDeletingMachineProviderID(t *testing.T) { + type testCase struct { + description string + providerID string + expectedReturn bool + } + + testCases := []testCase{ + { + description: "proper provider ID without deletion prefix should return false", + providerID: "fake-provider://a.provider.id-0001", + expectedReturn: false, + }, + { + description: "provider ID with deletion prefix should return true", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + expectedReturn: true, + }, + { + description: "empty provider ID should return false", + providerID: "", + expectedReturn: false, + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID returns true", + providerID: createDeletingMachineNormalizedProviderID("cluster-api", "id-0001"), + expectedReturn: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := isDeletingMachineProviderID(normalizedProviderID(tc.providerID)) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %t, observed %t", tc.providerID, tc.expectedReturn, observed) + } + }) + } + +} + +// TestNodeHasValidProviderID tests all permutations of provider IDs +// to determine whether the providerID is the standard cloud provider ID +// or has been modified by CAS CAPI provider +func TestNodeHasValidProviderID(t *testing.T) { + type testCase struct { + description string + providerID normalizedProviderID + expectedReturn bool + } + + testCases := []testCase{ + { + description: "real looking provider ID should return true", + providerID: normalizedProviderID("fake-provider://a.provider.id-0001"), + expectedReturn: true, + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createDeletingMachineNormalizedProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + { + description: "provider ID created with createPendingDeletionMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createPendingMachineProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + { + description: "provider ID created with createFailedMachineNormalizedProviderID should return false", + providerID: normalizedProviderID(createFailedMachineNormalizedProviderID("cluster-api", "id-0001")), + expectedReturn: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := isProviderIDNormalized(tc.providerID) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %t, observed %t", tc.providerID, tc.expectedReturn, observed) + } + }) + } +} + +func Test_machineKeyFromDeletingMachineProviderID(t *testing.T) { + type testCase struct { + description string + providerID string + expectedReturn string + } + + testCases := []testCase{ + { + description: "real looking provider ID with no deletion prefix returns provider id", + providerID: "fake-provider://a.provider.id-0001", + expectedReturn: "fake-provider://a.provider.id-0001", + }, + { + description: "namespace_name provider ID with no deletion prefix returns proper provider ID", + providerID: "cluster-api_id-0001", + expectedReturn: "cluster-api/id-0001", + }, + { + description: "namespace_name provider ID with deletion prefix returns proper provider ID", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + expectedReturn: "cluster-api/id-0001", + }, + { + description: "namespace_name provider ID with deletion prefix and two underscores returns proper provider ID", + providerID: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id_0001"), + expectedReturn: "cluster-api/id_0001", + }, + { + description: "provider ID created with createDeletingMachineNormalizedProviderID returns proper provider ID", + providerID: createDeletingMachineNormalizedProviderID("cluster-api", "id-0001"), + expectedReturn: "cluster-api/id-0001", + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := machineKeyFromDeletingMachineProviderID(normalizedProviderID(tc.providerID)) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for provider ID %q, expected %q, observed %q", tc.providerID, tc.expectedReturn, observed) + } + }) + } +} + +func Test_createDeletingMachineNormalizedProviderID(t *testing.T) { + type testCase struct { + description string + namespace string + name string + expectedReturn string + } + + testCases := []testCase{ + { + description: "namespace and name return proper normalized ID", + namespace: "cluster-api", + name: "id-0001", + expectedReturn: fmt.Sprintf("%s%s_%s", deletingMachinePrefix, "cluster-api", "id-0001"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := createDeletingMachineNormalizedProviderID(tc.namespace, tc.name) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for (namespace %q, name %q), expected %q, observed %q", tc.namespace, tc.name, tc.expectedReturn, observed) + } + }) + } +} + +// Test_createPendingMachineProviderID tests the creation of a pending machine provider ID +func Test_createPendingMachineProviderID(t *testing.T) { + type testCase struct { + description string + namespace string + name string + expectedReturn string + } + + testCases := []testCase{ + { + description: "namespace and name return proper normalized ID", + namespace: "cluster-api", + name: "id-0001", + expectedReturn: fmt.Sprintf("%s%s_%s", pendingMachinePrefix, "cluster-api", "id-0001"), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + observed := createPendingMachineProviderID(tc.namespace, tc.name) + if observed != tc.expectedReturn { + t.Fatalf("unexpected return for (namespace %q, name %q), expected %q, observed %q", tc.namespace, tc.name, tc.expectedReturn, observed) + } + }) + } +} diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go index c8204a9c8c6..acfda482fd6 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup.go @@ -203,11 +203,26 @@ func (ng *nodegroup) DecreaseTargetSize(delta int) error { return err } - if size+delta < len(nodes) { - return fmt.Errorf("attempt to delete existing nodes targetSize:%d delta:%d existingNodes: %d", - size, delta, len(nodes)) + // we want to filter out machines that are not nodes (eg failed or pending) + // so that the node group size can be set properly. this affects situations + // where an instance is created, but cannot become a node due to cloud provider + // issues such as quota limitations, and thus the autoscaler needs to properly + // set the size of the node group. without this adjustment, the core autoscaler + // will become confused about the state of nodes and instances in the clusterapi + // provider. + actualNodes := 0 + for _, node := range nodes { + if isProviderIDNormalized(normalizedProviderID(node.Id)) { + actualNodes += 1 + } + } + + if size+delta < actualNodes { + return fmt.Errorf("node group %s: attempt to delete existing nodes currentReplicas:%d delta:%d existingNodes: %d", + ng.scalableResource.Name(), size, delta, actualNodes) } + klog.V(4).Infof("%s: DecreaseTargetSize: scaling down: currentReplicas: %d, delta: %d, existingNodes: %d", ng.scalableResource.Name(), size, delta, len(nodes)) return ng.scalableResource.SetSize(size + delta) } diff --git a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go index 962ac11f5ed..75ec29805ba 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go +++ b/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_nodegroup_test.go @@ -406,18 +406,91 @@ func TestNodeGroupIncreaseSize(t *testing.T) { func TestNodeGroupDecreaseTargetSize(t *testing.T) { type testCase struct { - description string - delta int - initial int32 - targetSizeIncrement int32 - expected int32 - expectedError bool + description string + delta int + initial int32 + targetSizeIncrement int32 + expected int32 + expectedError bool + includeDeletingMachine bool + includeFailedMachine bool + includeFailedMachineWithProviderID bool + includePendingMachine bool + includePendingMachineWithProviderID bool + machinesDoNotHaveProviderIDs bool } test := func(t *testing.T, tc *testCase, testConfig *testConfig) { controller, stop := mustCreateTestController(t, testConfig) defer stop() + // machines in deletion should not be counted towards the active nodes when calculating a decrease in size. + if tc.includeDeletingMachine { + if tc.initial < 1 { + t.Fatal("test cannot pass, deleted machine requires at least 1 machine in machineset") + } + + // Simulate a machine in deleting + machine := testConfig.machines[0].DeepCopy() + timestamp := metav1.Now() + machine.SetDeletionTimestamp(×tamp) + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + + // machines that have failed should not be counted towards the active nodes when calculating a decrease in size. + if tc.includeFailedMachine { + // because we want to allow for tests that have deleted machines and failed machines, we use the second machine in the test data. + if tc.initial < 2 { + t.Fatal("test cannot pass, failed machine requires at least 2 machine in machineset") + } + + // Simulate a failed machine + machine := testConfig.machines[1].DeepCopy() + + if !tc.includeFailedMachineWithProviderID { + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + } + unstructured.SetNestedField(machine.Object, "FailureMessage", "status", "failureMessage") + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + + // machines that are in pending state should not be counted towards the active nodes when calculating a decrease in size. + if tc.includePendingMachine { + // because we want to allow for tests that have deleted, failed machines, and pending machine, we use the third machine in the test data. + if tc.initial < 3 { + t.Fatal("test cannot pass, pending machine requires at least 3 machine in machineset") + } + + // Simulate a pending machine + machine := testConfig.machines[2].DeepCopy() + + if !tc.includePendingMachineWithProviderID { + unstructured.RemoveNestedField(machine.Object, "spec", "providerID") + } + unstructured.RemoveNestedField(machine.Object, "status", "nodeRef") + + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, machine); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + + // machines with no provider id can be created on some providers, notably bare metal + if tc.machinesDoNotHaveProviderIDs { + for _, machine := range testConfig.machines { + updated := machine.DeepCopy() + unstructured.RemoveNestedField(updated.Object, "spec", "providerID") + if err := updateResource(controller.managementClient, controller.machineInformer, controller.machineResource, updated); err != nil { + t.Fatalf("unexpected error updating machine, got %v", err) + } + } + } + nodegroups, err := controller.nodeGroups() if err != nil { t.Fatalf("unexpected error: %v", err) @@ -522,45 +595,109 @@ func TestNodeGroupDecreaseTargetSize(t *testing.T) { } } - annotations := map[string]string{ - nodeGroupMinSizeAnnotationKey: "1", - nodeGroupMaxSizeAnnotationKey: "10", - } - - t.Run("MachineSet", func(t *testing.T) { - tc := testCase{ + testCases := []testCase{ + { description: "Same number of existing instances and node group target size should error", initial: 3, targetSizeIncrement: 0, expected: 3, delta: -1, expectedError: true, - } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) - - t.Run("MachineSet", func(t *testing.T) { - tc := testCase{ + }, + { description: "A node group with target size 4 but only 3 existing instances should decrease by 1", initial: 3, targetSizeIncrement: 1, expected: 3, delta: -1, - } - test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) + }, + { + description: "A node group with 4 replicas with one machine in deleting state should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeDeletingMachine: true, + }, + { + description: "A node group with 4 replicas with one failed machine should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeFailedMachine: true, + }, + { + description: "A node group with 4 replicas with one pending machine should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includePendingMachine: true, + }, + { + description: "A node group with 5 replicas with one pending and one failed machine should decrease by 2", + initial: 5, + targetSizeIncrement: 0, + expected: 3, + delta: -2, + includeFailedMachine: true, + includePendingMachine: true, + }, + { + description: "A node group with 5 replicas with one pending, one failed, and one deleting machine should decrease by 3", + initial: 5, + targetSizeIncrement: 0, + expected: 2, + delta: -3, + includeFailedMachine: true, + includePendingMachine: true, + includeDeletingMachine: true, + }, + { + description: "A node group with 4 replicas with one failed machine that has a provider ID should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includeFailedMachine: true, + includeFailedMachineWithProviderID: true, + }, + { + description: "A node group with 4 replicas with one pending machine that has a provider ID should decrease by 1", + initial: 4, + targetSizeIncrement: 0, + expected: 3, + delta: -1, + includePendingMachine: true, + includePendingMachineWithProviderID: true, + }, + { + description: "A node group with target size 4 but only 3 existing instances without provider IDs should not scale out", + initial: 3, + targetSizeIncrement: 1, + expected: 3, + delta: -1, + machinesDoNotHaveProviderIDs: true, + }, + } - t.Run("MachineDeployment", func(t *testing.T) { - tc := testCase{ - description: "Same number of existing instances and node group target size should error", - initial: 3, - targetSizeIncrement: 0, - expected: 3, - delta: -1, - expectedError: true, - } - test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) - }) + annotations := map[string]string{ + nodeGroupMinSizeAnnotationKey: "1", + nodeGroupMaxSizeAnnotationKey: "10", + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + test(t, &tc, createMachineSetTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + }) + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + test(t, &tc, createMachineDeploymentTestConfig(RandomString(6), RandomString(6), RandomString(6), int(tc.initial), annotations, nil)) + }) + } } func TestNodeGroupDecreaseSizeErrors(t *testing.T) { @@ -580,7 +717,7 @@ func TestNodeGroupDecreaseSizeErrors(t *testing.T) { description: "errors because initial+delta < len(nodes)", delta: -1, initial: 3, - errorMsg: "attempt to delete existing nodes targetSize:3 delta:-1 existingNodes: 3", + errorMsg: "attempt to delete existing nodes currentReplicas:3 delta:-1 existingNodes: 3", }} test := func(t *testing.T, tc *testCase, testConfig *testConfig) { diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/README.md b/cluster-autoscaler/cloudprovider/externalgrpc/README.md index c058e8c5bfe..d73118fc71f 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/README.md +++ b/cluster-autoscaler/cloudprovider/externalgrpc/README.md @@ -61,7 +61,12 @@ go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3 ``` -2. generate gRPC client and server code: +2. import proto dependencies using go modules +```bash +go mod vendor +``` + +3. generate gRPC client and server code: ```bash protoc \ diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go index ae57430d962..45bbad632c5 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/examples/external-grpc-cloud-provider-service/wrapper/wrapper.go @@ -103,7 +103,7 @@ func (w *Wrapper) NodeGroupForNode(_ context.Context, req *protos.NodeGroupForNo // Checks if ng is nil interface or contains nil value if ng == nil || reflect.ValueOf(ng).IsNil() { return &protos.NodeGroupForNodeResponse{ - NodeGroup: &protos.NodeGroup{}, //NodeGroup with id = "", meaning the node should not be processed by cluster autoscaler + NodeGroup: &protos.NodeGroup{}, // NodeGroup with id = "", meaning the node should not be processed by cluster autoscaler }, nil } return &protos.NodeGroupForNodeResponse{ @@ -365,6 +365,8 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu ScaleDownUnneededTime: pbDefaults.GetScaleDownUnneededTime().Duration, ScaleDownUnreadyTime: pbDefaults.GetScaleDownUnneededTime().Duration, MaxNodeProvisionTime: pbDefaults.GetMaxNodeProvisionTime().Duration, + ZeroOrMaxNodeScaling: pbDefaults.GetZeroOrMaxNodeScaling(), + IgnoreDaemonSetsUtilization: pbDefaults.GetIgnoreDaemonSetsUtilization(), } opts, err := ng.GetOptions(defaults) if err != nil { @@ -374,7 +376,7 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu return nil, err } if opts == nil { - return nil, fmt.Errorf("GetOptions not implemented") //make this explicitly so that grpc response is discarded + return nil, fmt.Errorf("GetOptions not implemented") // make this explicitly so that grpc response is discarded } return &protos.NodeGroupAutoscalingOptionsResponse{ NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ @@ -389,6 +391,8 @@ func (w *Wrapper) NodeGroupGetOptions(_ context.Context, req *protos.NodeGroupAu MaxNodeProvisionTime: &metav1.Duration{ Duration: opts.MaxNodeProvisionTime, }, + ZeroOrMaxNodeScaling: opts.ZeroOrMaxNodeScaling, + IgnoreDaemonSetsUtilization: opts.IgnoreDaemonSetsUtilization, }, }, nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go index 2f8000aded1..f62f3133eaa 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group.go @@ -283,6 +283,8 @@ func (n *NodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*co MaxNodeProvisionTime: &metav1.Duration{ Duration: defaults.MaxNodeProvisionTime, }, + ZeroOrMaxNodeScaling: defaults.ZeroOrMaxNodeScaling, + IgnoreDaemonSetsUtilization: defaults.IgnoreDaemonSetsUtilization, }, }) if err != nil { @@ -303,6 +305,8 @@ func (n *NodeGroup) GetOptions(defaults config.NodeGroupAutoscalingOptions) (*co ScaleDownUnneededTime: pbOpts.GetScaleDownUnneededTime().Duration, ScaleDownUnreadyTime: pbOpts.GetScaleDownUnreadyTime().Duration, MaxNodeProvisionTime: pbOpts.GetMaxNodeProvisionTime().Duration, + ZeroOrMaxNodeScaling: pbOpts.GetZeroOrMaxNodeScaling(), + IgnoreDaemonSetsUtilization: pbOpts.GetIgnoreDaemonSetsUtilization(), } return opts, nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go index 7dc27a4eac6..d4544e12d9c 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/externalgrpc_node_group_test.go @@ -244,7 +244,7 @@ func TestCloudProvider_GetOptions(t *testing.T) { client, m, teardown := setupTest(t) defer teardown() - // test correct call + // test correct call, NodeGroupAutoscalingOptionsResponse will override default options m.On( "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { return req.Id == "nodeGroup1" @@ -257,6 +257,8 @@ func TestCloudProvider_GetOptions(t *testing.T) { ScaleDownUnneededTime: &v1.Duration{Duration: time.Minute}, ScaleDownUnreadyTime: &v1.Duration{Duration: time.Hour}, MaxNodeProvisionTime: &v1.Duration{Duration: time.Minute}, + ZeroOrMaxNodeScaling: true, + IgnoreDaemonSetsUtilization: true, }, }, nil, @@ -267,12 +269,15 @@ func TestCloudProvider_GetOptions(t *testing.T) { client: client, grpcTimeout: defaultGRPCTimeout, } + defaultsOpts := config.NodeGroupAutoscalingOptions{ ScaleDownUtilizationThreshold: 0.6, ScaleDownGpuUtilizationThreshold: 0.7, ScaleDownUnneededTime: time.Minute, ScaleDownUnreadyTime: time.Hour, MaxNodeProvisionTime: time.Minute, + ZeroOrMaxNodeScaling: false, + IgnoreDaemonSetsUtilization: false, } opts, err := ng1.GetOptions(defaultsOpts) @@ -282,6 +287,8 @@ func TestCloudProvider_GetOptions(t *testing.T) { assert.Equal(t, time.Minute, opts.ScaleDownUnneededTime) assert.Equal(t, time.Hour, opts.ScaleDownUnreadyTime) assert.Equal(t, time.Minute, opts.MaxNodeProvisionTime) + assert.Equal(t, true, opts.ZeroOrMaxNodeScaling) + assert.Equal(t, true, opts.IgnoreDaemonSetsUtilization) // test grpc error m.On( @@ -289,9 +296,11 @@ func TestCloudProvider_GetOptions(t *testing.T) { return req.Id == "nodeGroup2" }), ).Return( - &protos.NodeGroupAutoscalingOptionsResponse{}, + &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{}, + }, fmt.Errorf("mock error"), - ) + ).Once() ng2 := NodeGroup{ id: "nodeGroup2", @@ -343,6 +352,40 @@ func TestCloudProvider_GetOptions(t *testing.T) { assert.Error(t, err) assert.Equal(t, cloudprovider.ErrNotImplemented, err) + // test with default options + m.On( + "NodeGroupGetOptions", mock.Anything, mock.MatchedBy(func(req *protos.NodeGroupAutoscalingOptionsRequest) bool { + return req.Id == "nodeGroup5" + }), + ).Return( + &protos.NodeGroupAutoscalingOptionsResponse{ + NodeGroupAutoscalingOptions: &protos.NodeGroupAutoscalingOptions{ + ScaleDownUtilizationThreshold: 0.6, + ScaleDownGpuUtilizationThreshold: 0.7, + ScaleDownUnneededTime: &v1.Duration{Duration: time.Minute}, + ScaleDownUnreadyTime: &v1.Duration{Duration: time.Hour}, + MaxNodeProvisionTime: &v1.Duration{Duration: time.Minute}, + }, + }, + nil, + ) + + ng5 := NodeGroup{ + id: "nodeGroup5", + client: client, + grpcTimeout: defaultGRPCTimeout, + } + + opts, err = ng5.GetOptions(defaultsOpts) + assert.NoError(t, err) + assert.Equal(t, 0.6, opts.ScaleDownUtilizationThreshold) + assert.Equal(t, 0.7, opts.ScaleDownGpuUtilizationThreshold) + assert.Equal(t, time.Minute, opts.ScaleDownUnneededTime) + assert.Equal(t, time.Hour, opts.ScaleDownUnreadyTime) + assert.Equal(t, time.Minute, opts.MaxNodeProvisionTime) + assert.Equal(t, false, opts.ZeroOrMaxNodeScaling) + assert.Equal(t, false, opts.IgnoreDaemonSetsUtilization) + } func TestCloudProvider_TargetSize(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go index 6bc0ceb3667..c500ba41f11 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.pb.go @@ -15,8 +15,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v3.21.12 +// protoc-gen-go v1.36.6 +// protoc v5.29.2 // source: cloudprovider/externalgrpc/protos/externalgrpc.proto package protos @@ -30,6 +30,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -96,10 +97,7 @@ func (InstanceStatus_InstanceState) EnumDescriptor() ([]byte, []int) { } type NodeGroup struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group on the cloud provider. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // MinSize of the node group on the cloud provider. @@ -107,16 +105,16 @@ type NodeGroup struct { // MaxSize of the node group on the cloud provider. MaxSize int32 `protobuf:"varint,3,opt,name=maxSize,proto3" json:"maxSize,omitempty"` // Debug returns a string containing all information regarding this node group. - Debug string `protobuf:"bytes,4,opt,name=debug,proto3" json:"debug,omitempty"` + Debug string `protobuf:"bytes,4,opt,name=debug,proto3" json:"debug,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroup) Reset() { *x = NodeGroup{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroup) String() string { @@ -127,7 +125,7 @@ func (*NodeGroup) ProtoMessage() {} func (x *NodeGroup) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -171,27 +169,24 @@ func (x *NodeGroup) GetDebug() string { } type ExternalGrpcNode struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node assigned by the cloud provider in the format: ://. ProviderID string `protobuf:"bytes,1,opt,name=providerID,proto3" json:"providerID,omitempty"` // Name of the node assigned by the cloud provider. Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` // labels is a map of {key,value} pairs with the node's labels. - Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // If specified, the node's annotations. - Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Annotations map[string]string `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExternalGrpcNode) Reset() { *x = ExternalGrpcNode{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExternalGrpcNode) String() string { @@ -202,7 +197,7 @@ func (*ExternalGrpcNode) ProtoMessage() {} func (x *ExternalGrpcNode) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -246,18 +241,16 @@ func (x *ExternalGrpcNode) GetAnnotations() map[string]string { } type NodeGroupsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupsRequest) Reset() { *x = NodeGroupsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupsRequest) String() string { @@ -268,7 +261,7 @@ func (*NodeGroupsRequest) ProtoMessage() {} func (x *NodeGroupsRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -284,21 +277,18 @@ func (*NodeGroupsRequest) Descriptor() ([]byte, []int) { } type NodeGroupsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // All the node groups that the cloud provider service supports. - NodeGroups []*NodeGroup `protobuf:"bytes,1,rep,name=nodeGroups,proto3" json:"nodeGroups,omitempty"` + NodeGroups []*NodeGroup `protobuf:"bytes,1,rep,name=nodeGroups,proto3" json:"nodeGroups,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupsResponse) Reset() { *x = NodeGroupsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupsResponse) String() string { @@ -309,7 +299,7 @@ func (*NodeGroupsResponse) ProtoMessage() {} func (x *NodeGroupsResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -332,21 +322,18 @@ func (x *NodeGroupsResponse) GetNodeGroups() []*NodeGroup { } type NodeGroupForNodeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node for which the request is performed. - Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupForNodeRequest) Reset() { *x = NodeGroupForNodeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupForNodeRequest) String() string { @@ -357,7 +344,7 @@ func (*NodeGroupForNodeRequest) ProtoMessage() {} func (x *NodeGroupForNodeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -380,21 +367,18 @@ func (x *NodeGroupForNodeRequest) GetNode() *ExternalGrpcNode { } type NodeGroupForNodeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node group for the given node. nodeGroup with id = "" means no node group. - NodeGroup *NodeGroup `protobuf:"bytes,1,opt,name=nodeGroup,proto3" json:"nodeGroup,omitempty"` + NodeGroup *NodeGroup `protobuf:"bytes,1,opt,name=nodeGroup,proto3" json:"nodeGroup,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupForNodeResponse) Reset() { *x = NodeGroupForNodeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupForNodeResponse) String() string { @@ -405,7 +389,7 @@ func (*NodeGroupForNodeResponse) ProtoMessage() {} func (x *NodeGroupForNodeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -428,25 +412,22 @@ func (x *NodeGroupForNodeResponse) GetNodeGroup() *NodeGroup { } type PricingNodePriceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Node for which the request is performed. Node *ExternalGrpcNode `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"` // Start time for the request period. StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` // End time for the request period. - EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingNodePriceRequest) Reset() { *x = PricingNodePriceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingNodePriceRequest) String() string { @@ -457,7 +438,7 @@ func (*PricingNodePriceRequest) ProtoMessage() {} func (x *PricingNodePriceRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -494,21 +475,18 @@ func (x *PricingNodePriceRequest) GetEndTime() *v1.Time { } type PricingNodePriceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Theoretical minimum price of running a node for a given period. - Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingNodePriceResponse) Reset() { *x = PricingNodePriceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingNodePriceResponse) String() string { @@ -519,7 +497,7 @@ func (*PricingNodePriceResponse) ProtoMessage() {} func (x *PricingNodePriceResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -542,25 +520,22 @@ func (x *PricingNodePriceResponse) GetPrice() float64 { } type PricingPodPriceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Pod for which the request is performed. Pod *v11.Pod `protobuf:"bytes,1,opt,name=pod,proto3" json:"pod,omitempty"` // Start time for the request period. StartTime *v1.Time `protobuf:"bytes,2,opt,name=startTime,proto3" json:"startTime,omitempty"` // End time for the request period. - EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + EndTime *v1.Time `protobuf:"bytes,3,opt,name=endTime,proto3" json:"endTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingPodPriceRequest) Reset() { *x = PricingPodPriceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingPodPriceRequest) String() string { @@ -571,7 +546,7 @@ func (*PricingPodPriceRequest) ProtoMessage() {} func (x *PricingPodPriceRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -608,21 +583,18 @@ func (x *PricingPodPriceRequest) GetEndTime() *v1.Time { } type PricingPodPriceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Theoretical minimum price of running a pod for a given period. - Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + Price float64 `protobuf:"fixed64,1,opt,name=price,proto3" json:"price,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *PricingPodPriceResponse) Reset() { *x = PricingPodPriceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *PricingPodPriceResponse) String() string { @@ -633,7 +605,7 @@ func (*PricingPodPriceResponse) ProtoMessage() {} func (x *PricingPodPriceResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -656,18 +628,16 @@ func (x *PricingPodPriceResponse) GetPrice() float64 { } type GPULabelRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPULabelRequest) Reset() { *x = GPULabelRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GPULabelRequest) String() string { @@ -678,7 +648,7 @@ func (*GPULabelRequest) ProtoMessage() {} func (x *GPULabelRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -694,21 +664,18 @@ func (*GPULabelRequest) Descriptor() ([]byte, []int) { } type GPULabelResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Label added to nodes with a GPU resource. - Label string `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"` + Label string `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPULabelResponse) Reset() { *x = GPULabelResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GPULabelResponse) String() string { @@ -719,7 +686,7 @@ func (*GPULabelResponse) ProtoMessage() {} func (x *GPULabelResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -742,18 +709,16 @@ func (x *GPULabelResponse) GetLabel() string { } type GetAvailableGPUTypesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetAvailableGPUTypesRequest) Reset() { *x = GetAvailableGPUTypesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetAvailableGPUTypesRequest) String() string { @@ -764,7 +729,7 @@ func (*GetAvailableGPUTypesRequest) ProtoMessage() {} func (x *GetAvailableGPUTypesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -780,21 +745,18 @@ func (*GetAvailableGPUTypesRequest) Descriptor() ([]byte, []int) { } type GetAvailableGPUTypesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // GPU types passed in as opaque key-value pairs. - GpuTypes map[string]*anypb.Any `protobuf:"bytes,1,rep,name=gpuTypes,proto3" json:"gpuTypes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + GpuTypes map[string]*anypb.Any `protobuf:"bytes,1,rep,name=gpuTypes,proto3" json:"gpuTypes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetAvailableGPUTypesResponse) Reset() { *x = GetAvailableGPUTypesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GetAvailableGPUTypesResponse) String() string { @@ -805,7 +767,7 @@ func (*GetAvailableGPUTypesResponse) ProtoMessage() {} func (x *GetAvailableGPUTypesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -828,18 +790,16 @@ func (x *GetAvailableGPUTypesResponse) GetGpuTypes() map[string]*anypb.Any { } type CleanupRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CleanupRequest) Reset() { *x = CleanupRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CleanupRequest) String() string { @@ -850,7 +810,7 @@ func (*CleanupRequest) ProtoMessage() {} func (x *CleanupRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -866,18 +826,16 @@ func (*CleanupRequest) Descriptor() ([]byte, []int) { } type CleanupResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CleanupResponse) Reset() { *x = CleanupResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CleanupResponse) String() string { @@ -888,7 +846,7 @@ func (*CleanupResponse) ProtoMessage() {} func (x *CleanupResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -904,18 +862,16 @@ func (*CleanupResponse) Descriptor() ([]byte, []int) { } type RefreshRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RefreshRequest) Reset() { *x = RefreshRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RefreshRequest) String() string { @@ -926,7 +882,7 @@ func (*RefreshRequest) ProtoMessage() {} func (x *RefreshRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -942,18 +898,16 @@ func (*RefreshRequest) Descriptor() ([]byte, []int) { } type RefreshResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RefreshResponse) Reset() { *x = RefreshResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *RefreshResponse) String() string { @@ -964,7 +918,7 @@ func (*RefreshResponse) ProtoMessage() {} func (x *RefreshResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -980,21 +934,18 @@ func (*RefreshResponse) Descriptor() ([]byte, []int) { } type NodeGroupTargetSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTargetSizeRequest) Reset() { *x = NodeGroupTargetSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTargetSizeRequest) String() string { @@ -1005,7 +956,7 @@ func (*NodeGroupTargetSizeRequest) ProtoMessage() {} func (x *NodeGroupTargetSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1028,21 +979,18 @@ func (x *NodeGroupTargetSizeRequest) GetId() string { } type NodeGroupTargetSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Current target size of the node group. - TargetSize int32 `protobuf:"varint,1,opt,name=targetSize,proto3" json:"targetSize,omitempty"` + TargetSize int32 `protobuf:"varint,1,opt,name=targetSize,proto3" json:"targetSize,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTargetSizeResponse) Reset() { *x = NodeGroupTargetSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTargetSizeResponse) String() string { @@ -1053,7 +1001,7 @@ func (*NodeGroupTargetSizeResponse) ProtoMessage() {} func (x *NodeGroupTargetSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1076,23 +1024,20 @@ func (x *NodeGroupTargetSizeResponse) GetTargetSize() int32 { } type NodeGroupIncreaseSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Number of nodes to add. Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupIncreaseSizeRequest) Reset() { *x = NodeGroupIncreaseSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupIncreaseSizeRequest) String() string { @@ -1103,7 +1048,7 @@ func (*NodeGroupIncreaseSizeRequest) ProtoMessage() {} func (x *NodeGroupIncreaseSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1133,18 +1078,16 @@ func (x *NodeGroupIncreaseSizeRequest) GetId() string { } type NodeGroupIncreaseSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupIncreaseSizeResponse) Reset() { *x = NodeGroupIncreaseSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupIncreaseSizeResponse) String() string { @@ -1155,7 +1098,7 @@ func (*NodeGroupIncreaseSizeResponse) ProtoMessage() {} func (x *NodeGroupIncreaseSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1171,23 +1114,20 @@ func (*NodeGroupIncreaseSizeResponse) Descriptor() ([]byte, []int) { } type NodeGroupDeleteNodesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // List of nodes to delete. Nodes []*ExternalGrpcNode `protobuf:"bytes,1,rep,name=nodes,proto3" json:"nodes,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDeleteNodesRequest) Reset() { *x = NodeGroupDeleteNodesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDeleteNodesRequest) String() string { @@ -1198,7 +1138,7 @@ func (*NodeGroupDeleteNodesRequest) ProtoMessage() {} func (x *NodeGroupDeleteNodesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1228,18 +1168,16 @@ func (x *NodeGroupDeleteNodesRequest) GetId() string { } type NodeGroupDeleteNodesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDeleteNodesResponse) Reset() { *x = NodeGroupDeleteNodesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDeleteNodesResponse) String() string { @@ -1250,7 +1188,7 @@ func (*NodeGroupDeleteNodesResponse) ProtoMessage() {} func (x *NodeGroupDeleteNodesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1266,23 +1204,20 @@ func (*NodeGroupDeleteNodesResponse) Descriptor() ([]byte, []int) { } type NodeGroupDecreaseTargetSizeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Number of nodes to delete. Delta int32 `protobuf:"varint,1,opt,name=delta,proto3" json:"delta,omitempty"` // ID of the node group for the request. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDecreaseTargetSizeRequest) Reset() { *x = NodeGroupDecreaseTargetSizeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDecreaseTargetSizeRequest) String() string { @@ -1293,7 +1228,7 @@ func (*NodeGroupDecreaseTargetSizeRequest) ProtoMessage() {} func (x *NodeGroupDecreaseTargetSizeRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1323,18 +1258,16 @@ func (x *NodeGroupDecreaseTargetSizeRequest) GetId() string { } type NodeGroupDecreaseTargetSizeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupDecreaseTargetSizeResponse) Reset() { *x = NodeGroupDecreaseTargetSizeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupDecreaseTargetSizeResponse) String() string { @@ -1345,7 +1278,7 @@ func (*NodeGroupDecreaseTargetSizeResponse) ProtoMessage() {} func (x *NodeGroupDecreaseTargetSizeResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1361,21 +1294,18 @@ func (*NodeGroupDecreaseTargetSizeResponse) Descriptor() ([]byte, []int) { } type NodeGroupNodesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupNodesRequest) Reset() { *x = NodeGroupNodesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupNodesRequest) String() string { @@ -1386,7 +1316,7 @@ func (*NodeGroupNodesRequest) ProtoMessage() {} func (x *NodeGroupNodesRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1409,21 +1339,18 @@ func (x *NodeGroupNodesRequest) GetId() string { } type NodeGroupNodesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // list of cloud provider instances in a node group. - Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` + Instances []*Instance `protobuf:"bytes,1,rep,name=instances,proto3" json:"instances,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupNodesResponse) Reset() { *x = NodeGroupNodesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupNodesResponse) String() string { @@ -1434,7 +1361,7 @@ func (*NodeGroupNodesResponse) ProtoMessage() {} func (x *NodeGroupNodesResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1457,23 +1384,20 @@ func (x *NodeGroupNodesResponse) GetInstances() []*Instance { } type Instance struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Id of the instance. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // Status of the node. - Status *InstanceStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + Status *InstanceStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Instance) Reset() { *x = Instance{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *Instance) String() string { @@ -1484,7 +1408,7 @@ func (*Instance) ProtoMessage() {} func (x *Instance) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1515,24 +1439,21 @@ func (x *Instance) GetStatus() *InstanceStatus { // InstanceStatus represents the instance status. type InstanceStatus struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // InstanceState tells if the instance is running, being created or being deleted. InstanceState InstanceStatus_InstanceState `protobuf:"varint,1,opt,name=instanceState,proto3,enum=clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus_InstanceState" json:"instanceState,omitempty"` // ErrorInfo provides information about the error status. // If there is no error condition related to instance, then errorInfo.errorCode should be an empty string. - ErrorInfo *InstanceErrorInfo `protobuf:"bytes,2,opt,name=errorInfo,proto3" json:"errorInfo,omitempty"` + ErrorInfo *InstanceErrorInfo `protobuf:"bytes,2,opt,name=errorInfo,proto3" json:"errorInfo,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceStatus) Reset() { *x = InstanceStatus{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InstanceStatus) String() string { @@ -1543,7 +1464,7 @@ func (*InstanceStatus) ProtoMessage() {} func (x *InstanceStatus) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1574,10 +1495,7 @@ func (x *InstanceStatus) GetErrorInfo() *InstanceErrorInfo { // InstanceErrorInfo provides information about error condition on instance. type InstanceErrorInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ErrorCode is cloud-provider specific error code for error condition. // An empty string for errorCode means there is no errorInfo for the instance (nil). ErrorCode string `protobuf:"bytes,1,opt,name=errorCode,proto3" json:"errorCode,omitempty"` @@ -1585,15 +1503,15 @@ type InstanceErrorInfo struct { ErrorMessage string `protobuf:"bytes,2,opt,name=errorMessage,proto3" json:"errorMessage,omitempty"` // InstanceErrorClass defines the class of error condition. InstanceErrorClass int32 `protobuf:"varint,3,opt,name=instanceErrorClass,proto3" json:"instanceErrorClass,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InstanceErrorInfo) Reset() { *x = InstanceErrorInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *InstanceErrorInfo) String() string { @@ -1604,7 +1522,7 @@ func (*InstanceErrorInfo) ProtoMessage() {} func (x *InstanceErrorInfo) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1641,21 +1559,18 @@ func (x *InstanceErrorInfo) GetInstanceErrorClass() int32 { } type NodeGroupTemplateNodeInfoRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTemplateNodeInfoRequest) Reset() { *x = NodeGroupTemplateNodeInfoRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTemplateNodeInfoRequest) String() string { @@ -1666,7 +1581,7 @@ func (*NodeGroupTemplateNodeInfoRequest) ProtoMessage() {} func (x *NodeGroupTemplateNodeInfoRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1689,21 +1604,18 @@ func (x *NodeGroupTemplateNodeInfoRequest) GetId() string { } type NodeGroupTemplateNodeInfoResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // nodeInfo is the extracted data from the cloud provider, as a primitive Kubernetes Node type. - NodeInfo *v11.Node `protobuf:"bytes,1,opt,name=nodeInfo,proto3" json:"nodeInfo,omitempty"` + NodeInfo *v11.Node `protobuf:"bytes,1,opt,name=nodeInfo,proto3" json:"nodeInfo,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupTemplateNodeInfoResponse) Reset() { *x = NodeGroupTemplateNodeInfoResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupTemplateNodeInfoResponse) String() string { @@ -1714,7 +1626,7 @@ func (*NodeGroupTemplateNodeInfoResponse) ProtoMessage() {} func (x *NodeGroupTemplateNodeInfoResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1737,10 +1649,7 @@ func (x *NodeGroupTemplateNodeInfoResponse) GetNodeInfo() *v11.Node { } type NodeGroupAutoscalingOptions struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // ScaleDownUtilizationThreshold sets threshold for nodes to be considered for scale down // if cpu or memory utilization is over threshold. ScaleDownUtilizationThreshold float64 `protobuf:"fixed64,1,opt,name=scaleDownUtilizationThreshold,proto3" json:"scaleDownUtilizationThreshold,omitempty"` @@ -1755,15 +1664,19 @@ type NodeGroupAutoscalingOptions struct { ScaleDownUnreadyTime *v1.Duration `protobuf:"bytes,4,opt,name=scaleDownUnreadyTime,proto3" json:"scaleDownUnreadyTime,omitempty"` // MaxNodeProvisionTime time CA waits for node to be provisioned MaxNodeProvisionTime *v1.Duration `protobuf:"bytes,5,opt,name=MaxNodeProvisionTime,proto3" json:"MaxNodeProvisionTime,omitempty"` + // ZeroOrMaxNodeScaling means that a node group should be scaled up to maximum size or down to zero nodes all at once instead of one-by-one. + ZeroOrMaxNodeScaling bool `protobuf:"varint,6,opt,name=zeroOrMaxNodeScaling,proto3" json:"zeroOrMaxNodeScaling,omitempty"` + // IgnoreDaemonSetsUtilization sets if daemonsets utilization should be considered during node scale-down + IgnoreDaemonSetsUtilization bool `protobuf:"varint,7,opt,name=ignoreDaemonSetsUtilization,proto3" json:"ignoreDaemonSetsUtilization,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptions) Reset() { *x = NodeGroupAutoscalingOptions{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptions) String() string { @@ -1774,7 +1687,7 @@ func (*NodeGroupAutoscalingOptions) ProtoMessage() {} func (x *NodeGroupAutoscalingOptions) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1824,24 +1737,35 @@ func (x *NodeGroupAutoscalingOptions) GetMaxNodeProvisionTime() *v1.Duration { return nil } -type NodeGroupAutoscalingOptionsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *NodeGroupAutoscalingOptions) GetZeroOrMaxNodeScaling() bool { + if x != nil { + return x.ZeroOrMaxNodeScaling + } + return false +} + +func (x *NodeGroupAutoscalingOptions) GetIgnoreDaemonSetsUtilization() bool { + if x != nil { + return x.IgnoreDaemonSetsUtilization + } + return false +} +type NodeGroupAutoscalingOptionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` // ID of the node group for the request. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // default node group autoscaling options. - Defaults *NodeGroupAutoscalingOptions `protobuf:"bytes,2,opt,name=defaults,proto3" json:"defaults,omitempty"` + Defaults *NodeGroupAutoscalingOptions `protobuf:"bytes,2,opt,name=defaults,proto3" json:"defaults,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptionsRequest) Reset() { *x = NodeGroupAutoscalingOptionsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptionsRequest) String() string { @@ -1852,7 +1776,7 @@ func (*NodeGroupAutoscalingOptionsRequest) ProtoMessage() {} func (x *NodeGroupAutoscalingOptionsRequest) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1882,21 +1806,18 @@ func (x *NodeGroupAutoscalingOptionsRequest) GetDefaults() *NodeGroupAutoscaling } type NodeGroupAutoscalingOptionsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // autoscaling options for the requested node. NodeGroupAutoscalingOptions *NodeGroupAutoscalingOptions `protobuf:"bytes,1,opt,name=nodeGroupAutoscalingOptions,proto3" json:"nodeGroupAutoscalingOptions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NodeGroupAutoscalingOptionsResponse) Reset() { *x = NodeGroupAutoscalingOptionsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *NodeGroupAutoscalingOptionsResponse) String() string { @@ -1907,7 +1828,7 @@ func (*NodeGroupAutoscalingOptionsResponse) ProtoMessage() {} func (x *NodeGroupAutoscalingOptionsResponse) ProtoReflect() protoreflect.Message { mi := &file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1931,463 +1852,148 @@ func (x *NodeGroupAutoscalingOptionsResponse) GetNodeGroupAutoscalingOptions() * var File_cloudprovider_externalgrpc_protos_externalgrpc_proto protoreflect.FileDescriptor -var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = []byte{ - 0x0a, 0x34, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x34, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, - 0x69, 0x73, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, 0x6b, 0x38, 0x73, 0x2e, - 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x65, - 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x6d, - 0x69, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x69, - 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x64, 0x65, 0x62, 0x75, 0x67, 0x22, 0x9e, 0x03, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x49, 0x44, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x65, - 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4d, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, - 0x65, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x74, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x52, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x6e, - 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, - 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x13, 0x0a, 0x11, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x70, 0x0a, 0x12, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x5a, 0x0a, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x52, 0x0a, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x70, 0x0a, - 0x17, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, - 0x74, 0x0a, 0x18, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, - 0x6f, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x6e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x09, 0x6e, 0x6f, 0x64, 0x65, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, - 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x55, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, - 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, - 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, - 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x30, 0x0a, 0x18, 0x50, 0x72, 0x69, 0x63, - 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, 0x65, 0x22, 0xd3, 0x01, 0x0a, 0x16, 0x50, - 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x03, 0x70, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x03, 0x70, 0x6f, 0x64, - 0x12, 0x48, 0x0a, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, - 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x44, 0x0a, 0x07, 0x65, 0x6e, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6b, 0x38, - 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, - 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, - 0x22, 0x2f, 0x0a, 0x17, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x70, - 0x72, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x70, 0x72, 0x69, 0x63, - 0x65, 0x22, 0x11, 0x0a, 0x0f, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x28, 0x0a, 0x10, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x22, 0x1d, - 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, - 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xea, 0x01, - 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, - 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, - 0x0a, 0x08, 0x67, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x5b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, - 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x47, 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x67, - 0x70, 0x75, 0x54, 0x79, 0x70, 0x65, 0x73, 0x1a, 0x51, 0x0a, 0x0d, 0x47, 0x70, 0x75, 0x54, 0x79, - 0x70, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x10, 0x0a, 0x0e, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x11, 0x0a, 0x0f, - 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x10, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2c, 0x0a, 0x1a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x22, 0x3d, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, - 0x65, 0x22, 0x44, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, - 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1f, 0x0a, 0x1d, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, - 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x57, 0x0a, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x47, 0x72, 0x70, 0x63, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x6e, 0x6f, 0x64, 0x65, - 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x22, 0x1e, 0x0a, 0x1c, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x4a, 0x0a, 0x22, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, - 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x25, 0x0a, - 0x23, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, - 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x27, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x71, 0x0a, - 0x16, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, - 0x22, 0x73, 0x0a, 0x08, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x57, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0xca, 0x02, 0x0a, 0x0e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x73, 0x0a, 0x0d, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x60, 0x0a, - 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x22, - 0x61, 0x0a, 0x0d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x0f, 0x0a, 0x0b, 0x75, 0x6e, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x64, 0x10, - 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x75, 0x6e, - 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, 0x14, 0x0a, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x14, 0x0a, 0x10, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x69, 0x6e, 0x67, - 0x10, 0x03, 0x22, 0x85, 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, - 0x72, 0x72, 0x6f, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x43, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x22, 0x32, 0x0a, 0x20, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, - 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x59, - 0x0a, 0x21, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, - 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, - 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, - 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xdd, 0x03, 0x0a, 0x1b, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, - 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x44, 0x0a, 0x1d, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, - 0x52, 0x1d, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x74, 0x69, 0x6c, 0x69, - 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, - 0x4a, 0x0a, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, - 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, - 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x20, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x44, 0x6f, 0x77, 0x6e, 0x47, 0x70, 0x75, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x64, 0x0a, 0x15, 0x73, - 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6b, 0x38, 0x73, - 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, - 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x15, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x6e, 0x65, 0x65, 0x64, 0x65, 0x64, 0x54, 0x69, 0x6d, - 0x65, 0x12, 0x62, 0x0a, 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2e, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x6d, 0x61, 0x63, 0x68, - 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, 0x69, 0x73, 0x2e, 0x6d, - 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x14, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x55, 0x6e, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x62, 0x0a, 0x14, 0x4d, 0x61, 0x78, 0x4e, 0x6f, 0x64, 0x65, - 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x61, 0x70, - 0x69, 0x73, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x4d, 0x61, 0x78, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x9e, 0x01, 0x0a, 0x22, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, - 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x68, 0x0a, 0x08, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, - 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x52, 0x08, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x73, 0x22, 0xb6, 0x01, 0x0a, 0x23, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x8e, 0x01, 0x0a, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x1b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x32, 0xbf, 0x14, 0x0a, 0x0d, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x50, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x97, 0x01, 0x0a, 0x0a, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x12, 0x42, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x43, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0xa9, 0x01, 0x0a, 0x10, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, - 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x46, 0x6f, 0x72, 0x4e, 0x6f, 0x64, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa9, 0x01, 0x0a, 0x10, - 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, - 0x12, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, - 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, - 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, - 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x49, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, - 0x63, 0x69, 0x6e, 0x67, 0x4e, 0x6f, 0x64, 0x65, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xa6, 0x01, 0x0a, 0x0f, 0x50, 0x72, 0x69, 0x63, - 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x12, 0x47, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, - 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x48, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, - 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, - 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x72, 0x69, 0x63, 0x69, 0x6e, 0x67, 0x50, 0x6f, - 0x64, 0x50, 0x72, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0x91, 0x01, 0x0a, 0x08, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x40, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x41, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x47, 0x50, 0x55, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0xb5, 0x01, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x4c, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x47, 0x65, 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, - 0x79, 0x70, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x47, 0x65, - 0x74, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x47, 0x50, 0x55, 0x54, 0x79, 0x70, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, 0x0a, - 0x07, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, - 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6c, 0x65, 0x61, - 0x6e, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x8e, 0x01, - 0x0a, 0x07, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x12, 0x3f, 0x2e, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, 0x72, - 0x65, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x40, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x66, - 0x72, 0x65, 0x73, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb2, - 0x01, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4b, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, - 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0xb8, 0x01, 0x0a, 0x15, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4d, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4e, 0x2e, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, - 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, - 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, - 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x6e, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, - 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xb5, - 0x01, 0x0a, 0x14, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x4c, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4d, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, - 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, - 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xca, 0x01, 0x0a, 0x1b, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, 0x63, 0x6c, - 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, - 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, - 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x44, 0x65, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0xa3, 0x01, 0x0a, 0x0e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x46, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x47, - 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, - 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0xc4, 0x01, 0x0a, 0x19, 0x4e, 0x6f, - 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, - 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x51, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x52, 0x2e, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, - 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x6f, - 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, - 0x12, 0xc2, 0x01, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x47, 0x65, - 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x54, 0x2e, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, - 0x72, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2e, - 0x76, 0x31, 0x2e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x41, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, - 0x6c, 0x69, 0x6e, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x36, 0x5a, 0x34, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x2d, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2f, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x2f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = "" + + "\n" + + "4cloudprovider/externalgrpc/protos/externalgrpc.proto\x12/clusterautoscaler.cloudprovider.v1.externalgrpc\x1a\x19google/protobuf/any.proto\x1a google/protobuf/descriptor.proto\x1a\"k8s.io/api/core/v1/generated.proto\x1a4k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto\"e\n" + + "\tNodeGroup\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + + "\aminSize\x18\x02 \x01(\x05R\aminSize\x12\x18\n" + + "\amaxSize\x18\x03 \x01(\x05R\amaxSize\x12\x14\n" + + "\x05debug\x18\x04 \x01(\tR\x05debug\"\x9e\x03\n" + + "\x10ExternalGrpcNode\x12\x1e\n" + + "\n" + + "providerID\x18\x01 \x01(\tR\n" + + "providerID\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12e\n" + + "\x06labels\x18\x03 \x03(\v2M.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.LabelsEntryR\x06labels\x12t\n" + + "\vannotations\x18\x04 \x03(\v2R.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode.AnnotationsEntryR\vannotations\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a>\n" + + "\x10AnnotationsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x13\n" + + "\x11NodeGroupsRequest\"p\n" + + "\x12NodeGroupsResponse\x12Z\n" + + "\n" + + "nodeGroups\x18\x01 \x03(\v2:.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupR\n" + + "nodeGroups\"p\n" + + "\x17NodeGroupForNodeRequest\x12U\n" + + "\x04node\x18\x01 \x01(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x04node\"t\n" + + "\x18NodeGroupForNodeResponse\x12X\n" + + "\tnodeGroup\x18\x01 \x01(\v2:.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupR\tnodeGroup\"\x80\x02\n" + + "\x17PricingNodePriceRequest\x12U\n" + + "\x04node\x18\x01 \x01(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x04node\x12H\n" + + "\tstartTime\x18\x02 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\tstartTime\x12D\n" + + "\aendTime\x18\x03 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\aendTime\"0\n" + + "\x18PricingNodePriceResponse\x12\x14\n" + + "\x05price\x18\x01 \x01(\x01R\x05price\"\xd3\x01\n" + + "\x16PricingPodPriceRequest\x12)\n" + + "\x03pod\x18\x01 \x01(\v2\x17.k8s.io.api.core.v1.PodR\x03pod\x12H\n" + + "\tstartTime\x18\x02 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\tstartTime\x12D\n" + + "\aendTime\x18\x03 \x01(\v2*.k8s.io.apimachinery.pkg.apis.meta.v1.TimeR\aendTime\"/\n" + + "\x17PricingPodPriceResponse\x12\x14\n" + + "\x05price\x18\x01 \x01(\x01R\x05price\"\x11\n" + + "\x0fGPULabelRequest\"(\n" + + "\x10GPULabelResponse\x12\x14\n" + + "\x05label\x18\x01 \x01(\tR\x05label\"\x1d\n" + + "\x1bGetAvailableGPUTypesRequest\"\xea\x01\n" + + "\x1cGetAvailableGPUTypesResponse\x12w\n" + + "\bgpuTypes\x18\x01 \x03(\v2[.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse.GpuTypesEntryR\bgpuTypes\x1aQ\n" + + "\rGpuTypesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12*\n" + + "\x05value\x18\x02 \x01(\v2\x14.google.protobuf.AnyR\x05value:\x028\x01\"\x10\n" + + "\x0eCleanupRequest\"\x11\n" + + "\x0fCleanupResponse\"\x10\n" + + "\x0eRefreshRequest\"\x11\n" + + "\x0fRefreshResponse\",\n" + + "\x1aNodeGroupTargetSizeRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"=\n" + + "\x1bNodeGroupTargetSizeResponse\x12\x1e\n" + + "\n" + + "targetSize\x18\x01 \x01(\x05R\n" + + "targetSize\"D\n" + + "\x1cNodeGroupIncreaseSizeRequest\x12\x14\n" + + "\x05delta\x18\x01 \x01(\x05R\x05delta\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"\x1f\n" + + "\x1dNodeGroupIncreaseSizeResponse\"\x86\x01\n" + + "\x1bNodeGroupDeleteNodesRequest\x12W\n" + + "\x05nodes\x18\x01 \x03(\v2A.clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNodeR\x05nodes\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"\x1e\n" + + "\x1cNodeGroupDeleteNodesResponse\"J\n" + + "\"NodeGroupDecreaseTargetSizeRequest\x12\x14\n" + + "\x05delta\x18\x01 \x01(\x05R\x05delta\x12\x0e\n" + + "\x02id\x18\x02 \x01(\tR\x02id\"%\n" + + "#NodeGroupDecreaseTargetSizeResponse\"'\n" + + "\x15NodeGroupNodesRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"q\n" + + "\x16NodeGroupNodesResponse\x12W\n" + + "\tinstances\x18\x01 \x03(\v29.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceR\tinstances\"s\n" + + "\bInstance\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12W\n" + + "\x06status\x18\x02 \x01(\v2?.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatusR\x06status\"\xca\x02\n" + + "\x0eInstanceStatus\x12s\n" + + "\rinstanceState\x18\x01 \x01(\x0e2M.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceStateR\rinstanceState\x12`\n" + + "\terrorInfo\x18\x02 \x01(\v2B.clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceErrorInfoR\terrorInfo\"a\n" + + "\rInstanceState\x12\x0f\n" + + "\vunspecified\x10\x00\x12\x13\n" + + "\x0finstanceRunning\x10\x01\x12\x14\n" + + "\x10instanceCreating\x10\x02\x12\x14\n" + + "\x10instanceDeleting\x10\x03\"\x85\x01\n" + + "\x11InstanceErrorInfo\x12\x1c\n" + + "\terrorCode\x18\x01 \x01(\tR\terrorCode\x12\"\n" + + "\ferrorMessage\x18\x02 \x01(\tR\ferrorMessage\x12.\n" + + "\x12instanceErrorClass\x18\x03 \x01(\x05R\x12instanceErrorClass\"2\n" + + " NodeGroupTemplateNodeInfoRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"Y\n" + + "!NodeGroupTemplateNodeInfoResponse\x124\n" + + "\bnodeInfo\x18\x01 \x01(\v2\x18.k8s.io.api.core.v1.NodeR\bnodeInfo\"\xd3\x04\n" + + "\x1bNodeGroupAutoscalingOptions\x12D\n" + + "\x1dscaleDownUtilizationThreshold\x18\x01 \x01(\x01R\x1dscaleDownUtilizationThreshold\x12J\n" + + " scaleDownGpuUtilizationThreshold\x18\x02 \x01(\x01R scaleDownGpuUtilizationThreshold\x12d\n" + + "\x15scaleDownUnneededTime\x18\x03 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x15scaleDownUnneededTime\x12b\n" + + "\x14scaleDownUnreadyTime\x18\x04 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x14scaleDownUnreadyTime\x12b\n" + + "\x14MaxNodeProvisionTime\x18\x05 \x01(\v2..k8s.io.apimachinery.pkg.apis.meta.v1.DurationR\x14MaxNodeProvisionTime\x122\n" + + "\x14zeroOrMaxNodeScaling\x18\x06 \x01(\bR\x14zeroOrMaxNodeScaling\x12@\n" + + "\x1bignoreDaemonSetsUtilization\x18\a \x01(\bR\x1bignoreDaemonSetsUtilization\"\x9e\x01\n" + + "\"NodeGroupAutoscalingOptionsRequest\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12h\n" + + "\bdefaults\x18\x02 \x01(\v2L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsR\bdefaults\"\xb6\x01\n" + + "#NodeGroupAutoscalingOptionsResponse\x12\x8e\x01\n" + + "\x1bnodeGroupAutoscalingOptions\x18\x01 \x01(\v2L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsR\x1bnodeGroupAutoscalingOptions2\xbf\x14\n" + + "\rCloudProvider\x12\x97\x01\n" + + "\n" + + "NodeGroups\x12B.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsRequest\x1aC.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupsResponse\"\x00\x12\xa9\x01\n" + + "\x10NodeGroupForNode\x12H.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeRequest\x1aI.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupForNodeResponse\"\x00\x12\xa9\x01\n" + + "\x10PricingNodePrice\x12H.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceRequest\x1aI.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingNodePriceResponse\"\x00\x12\xa6\x01\n" + + "\x0fPricingPodPrice\x12G.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceRequest\x1aH.clusterautoscaler.cloudprovider.v1.externalgrpc.PricingPodPriceResponse\"\x00\x12\x91\x01\n" + + "\bGPULabel\x12@.clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelRequest\x1aA.clusterautoscaler.cloudprovider.v1.externalgrpc.GPULabelResponse\"\x00\x12\xb5\x01\n" + + "\x14GetAvailableGPUTypes\x12L.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesRequest\x1aM.clusterautoscaler.cloudprovider.v1.externalgrpc.GetAvailableGPUTypesResponse\"\x00\x12\x8e\x01\n" + + "\aCleanup\x12?.clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupRequest\x1a@.clusterautoscaler.cloudprovider.v1.externalgrpc.CleanupResponse\"\x00\x12\x8e\x01\n" + + "\aRefresh\x12?.clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshRequest\x1a@.clusterautoscaler.cloudprovider.v1.externalgrpc.RefreshResponse\"\x00\x12\xb2\x01\n" + + "\x13NodeGroupTargetSize\x12K.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeRequest\x1aL.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTargetSizeResponse\"\x00\x12\xb8\x01\n" + + "\x15NodeGroupIncreaseSize\x12M.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeRequest\x1aN.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupIncreaseSizeResponse\"\x00\x12\xb5\x01\n" + + "\x14NodeGroupDeleteNodes\x12L.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesRequest\x1aM.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDeleteNodesResponse\"\x00\x12\xca\x01\n" + + "\x1bNodeGroupDecreaseTargetSize\x12S.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeRequest\x1aT.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupDecreaseTargetSizeResponse\"\x00\x12\xa3\x01\n" + + "\x0eNodeGroupNodes\x12F.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesRequest\x1aG.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupNodesResponse\"\x00\x12\xc4\x01\n" + + "\x19NodeGroupTemplateNodeInfo\x12Q.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoRequest\x1aR.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupTemplateNodeInfoResponse\"\x00\x12\xc2\x01\n" + + "\x13NodeGroupGetOptions\x12S.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsRequest\x1aT.clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroupAutoscalingOptionsResponse\"\x00B6Z4cluster-autoscaler/cloudprovider/externalgrpc/protosb\x06proto3" var ( file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce sync.Once - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData []byte ) func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescGZIP() []byte { file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescOnce.Do(func() { - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData) + file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc), len(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc))) }) return file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDescData } var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes = make([]protoimpl.MessageInfo, 39) -var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = []interface{}{ +var file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = []any{ (InstanceStatus_InstanceState)(0), // 0: clusterautoscaler.cloudprovider.v1.externalgrpc.InstanceStatus.InstanceState (*NodeGroup)(nil), // 1: clusterautoscaler.cloudprovider.v1.externalgrpc.NodeGroup (*ExternalGrpcNode)(nil), // 2: clusterautoscaler.cloudprovider.v1.externalgrpc.ExternalGrpcNode @@ -2501,445 +2107,11 @@ func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() { if File_cloudprovider_externalgrpc_protos_externalgrpc_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroup); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExternalGrpcNode); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupForNodeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupForNodeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingNodePriceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingNodePriceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingPodPriceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PricingPodPriceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GPULabelRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GPULabelResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetAvailableGPUTypesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetAvailableGPUTypesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanupRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanupResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefreshRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefreshResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTargetSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTargetSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupIncreaseSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupIncreaseSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDeleteNodesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDeleteNodesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDecreaseTargetSizeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupDecreaseTargetSizeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupNodesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupNodesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Instance); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstanceStatus); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstanceErrorInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTemplateNodeInfoRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupTemplateNodeInfoResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptions); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptionsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NodeGroupAutoscalingOptionsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc), len(file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc)), NumEnums: 1, NumMessages: 39, NumExtensions: 0, @@ -2951,7 +2123,6 @@ func file_cloudprovider_externalgrpc_protos_externalgrpc_proto_init() { MessageInfos: file_cloudprovider_externalgrpc_protos_externalgrpc_proto_msgTypes, }.Build() File_cloudprovider_externalgrpc_protos_externalgrpc_proto = out.File - file_cloudprovider_externalgrpc_protos_externalgrpc_proto_rawDesc = nil file_cloudprovider_externalgrpc_protos_externalgrpc_proto_goTypes = nil file_cloudprovider_externalgrpc_protos_externalgrpc_proto_depIdxs = nil } diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto index 0a476e65973..0e87886666d 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc.proto @@ -1,28 +1,27 @@ /* -Copyright 2022 The Kubernetes Authors. + Copyright 2022 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ syntax = "proto3"; package clusterautoscaler.cloudprovider.v1.externalgrpc; -import "google/protobuf/descriptor.proto"; import "google/protobuf/any.proto"; - -import "k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto"; +import "google/protobuf/descriptor.proto"; import "k8s.io/api/core/v1/generated.proto"; +import "k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto"; option go_package = "cluster-autoscaler/cloudprovider/externalgrpc/protos"; @@ -30,42 +29,34 @@ service CloudProvider { // CloudProvider specific RPC functions // NodeGroups returns all node groups configured for this cloud provider. - rpc NodeGroups(NodeGroupsRequest) - returns (NodeGroupsResponse) {} + rpc NodeGroups(NodeGroupsRequest) returns (NodeGroupsResponse) {} // NodeGroupForNode returns the node group for the given node. // The node group id is an empty string if the node should not // be processed by cluster autoscaler. - rpc NodeGroupForNode(NodeGroupForNodeRequest) - returns (NodeGroupForNodeResponse) {} + rpc NodeGroupForNode(NodeGroupForNodeRequest) returns (NodeGroupForNodeResponse) {} // PricingNodePrice returns a theoretical minimum price of running a node for // a given period of time on a perfectly matching machine. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc PricingNodePrice(PricingNodePriceRequest) - returns (PricingNodePriceResponse) {} + rpc PricingNodePrice(PricingNodePriceRequest) returns (PricingNodePriceResponse) {} // PricingPodPrice returns a theoretical minimum price of running a pod for a given // period of time on a perfectly matching machine. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc PricingPodPrice(PricingPodPriceRequest) - returns (PricingPodPriceResponse) {} + rpc PricingPodPrice(PricingPodPriceRequest) returns (PricingPodPriceResponse) {} // GPULabel returns the label added to nodes with GPU resource. - rpc GPULabel(GPULabelRequest) - returns (GPULabelResponse) {} + rpc GPULabel(GPULabelRequest) returns (GPULabelResponse) {} // GetAvailableGPUTypes return all available GPU types cloud provider supports. - rpc GetAvailableGPUTypes(GetAvailableGPUTypesRequest) - returns (GetAvailableGPUTypesResponse) {} + rpc GetAvailableGPUTypes(GetAvailableGPUTypesRequest) returns (GetAvailableGPUTypesResponse) {} // Cleanup cleans up open resources before the cloud provider is destroyed, i.e. go routines etc. - rpc Cleanup(CleanupRequest) - returns (CleanupResponse) {} + rpc Cleanup(CleanupRequest) returns (CleanupResponse) {} // Refresh is called before every main loop and can be used to dynamically update cloud provider state. - rpc Refresh(RefreshRequest) - returns (RefreshResponse) {} + rpc Refresh(RefreshRequest) returns (RefreshResponse) {} // NodeGroup specific RPC functions @@ -73,45 +64,38 @@ service CloudProvider { // that the number of nodes in Kubernetes is different at the moment but should be equal // to the size of a node group once everything stabilizes (new nodes finish startup and // registration or removed nodes are deleted completely). - rpc NodeGroupTargetSize(NodeGroupTargetSizeRequest) - returns (NodeGroupTargetSizeResponse) {} + rpc NodeGroupTargetSize(NodeGroupTargetSizeRequest) returns (NodeGroupTargetSizeResponse) {} // NodeGroupIncreaseSize increases the size of the node group. To delete a node you need // to explicitly name it and use NodeGroupDeleteNodes. This function should wait until // node group size is updated. - rpc NodeGroupIncreaseSize(NodeGroupIncreaseSizeRequest) - returns (NodeGroupIncreaseSizeResponse) {} + rpc NodeGroupIncreaseSize(NodeGroupIncreaseSizeRequest) returns (NodeGroupIncreaseSizeResponse) {} // NodeGroupDeleteNodes deletes nodes from this node group (and also decreasing the size // of the node group with that). Error is returned either on failure or if the given node // doesn't belong to this node group. This function should wait until node group size is updated. - rpc NodeGroupDeleteNodes(NodeGroupDeleteNodesRequest) - returns (NodeGroupDeleteNodesResponse) {} + rpc NodeGroupDeleteNodes(NodeGroupDeleteNodesRequest) returns (NodeGroupDeleteNodesResponse) {} // NodeGroupDecreaseTargetSize decreases the target size of the node group. This function // doesn't permit to delete any existing node and can be used only to reduce the request // for new nodes that have not been yet fulfilled. Delta should be negative. It is assumed // that cloud provider will not delete the existing nodes if the size when there is an option // to just decrease the target. - rpc NodeGroupDecreaseTargetSize(NodeGroupDecreaseTargetSizeRequest) - returns (NodeGroupDecreaseTargetSizeResponse) {} + rpc NodeGroupDecreaseTargetSize(NodeGroupDecreaseTargetSizeRequest) returns (NodeGroupDecreaseTargetSizeResponse) {} // NodeGroupNodes returns a list of all nodes that belong to this node group. - rpc NodeGroupNodes(NodeGroupNodesRequest) - returns (NodeGroupNodesResponse) {} + rpc NodeGroupNodes(NodeGroupNodesRequest) returns (NodeGroupNodesResponse) {} // NodeGroupTemplateNodeInfo returns a structure of an empty (as if just started) node, // with all of the labels, capacity and allocatable information. This will be used in // scale-up simulations to predict what would a new node look like if a node group was expanded. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc NodeGroupTemplateNodeInfo(NodeGroupTemplateNodeInfoRequest) - returns (NodeGroupTemplateNodeInfoResponse) {} + rpc NodeGroupTemplateNodeInfo(NodeGroupTemplateNodeInfoRequest) returns (NodeGroupTemplateNodeInfoResponse) {} // GetOptions returns NodeGroupAutoscalingOptions that should be used for this particular // NodeGroup. // Implementation optional: if unimplemented return error code 12 (for `Unimplemented`) - rpc NodeGroupGetOptions(NodeGroupAutoscalingOptionsRequest) - returns (NodeGroupAutoscalingOptionsResponse) {} + rpc NodeGroupGetOptions(NodeGroupAutoscalingOptionsRequest) returns (NodeGroupAutoscalingOptionsResponse) {} } message NodeGroup { @@ -128,7 +112,7 @@ message NodeGroup { string debug = 4; } -message ExternalGrpcNode{ +message ExternalGrpcNode { // ID of the node assigned by the cloud provider in the format: ://. string providerID = 1; @@ -259,7 +243,7 @@ message NodeGroupDeleteNodesRequest { message NodeGroupDeleteNodesResponse { // Intentionally empty. - } +} message NodeGroupDecreaseTargetSizeRequest { // Number of nodes to delete. @@ -343,7 +327,7 @@ message NodeGroupAutoscalingOptions { // if cpu or memory utilization is over threshold. double scaleDownUtilizationThreshold = 1; - // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be + // ScaleDownGpuUtilizationThreshold sets threshold for gpu nodes to be // considered for scale down if gpu utilization is over threshold. double scaleDownGpuUtilizationThreshold = 2; @@ -356,7 +340,13 @@ message NodeGroupAutoscalingOptions { k8s.io.apimachinery.pkg.apis.meta.v1.Duration scaleDownUnreadyTime = 4; // MaxNodeProvisionTime time CA waits for node to be provisioned - k8s.io.apimachinery.pkg.apis.meta.v1.Duration MaxNodeProvisionTime = 5; + k8s.io.apimachinery.pkg.apis.meta.v1.Duration MaxNodeProvisionTime = 5; + + // ZeroOrMaxNodeScaling means that a node group should be scaled up to maximum size or down to zero nodes all at once instead of one-by-one. + bool zeroOrMaxNodeScaling = 6; + + // IgnoreDaemonSetsUtilization sets if daemonsets utilization should be considered during node scale-down + bool ignoreDaemonSetsUtilization = 7; } message NodeGroupAutoscalingOptionsRequest { diff --git a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go index 1b2b4f38a37..e788f5d071e 100644 --- a/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go +++ b/cluster-autoscaler/cloudprovider/externalgrpc/protos/externalgrpc_grpc.pb.go @@ -15,8 +15,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.21.12 +// - protoc-gen-go-grpc v1.5.1 +// - protoc v5.29.2 // source: cloudprovider/externalgrpc/protos/externalgrpc.proto package protos @@ -30,8 +30,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 const ( CloudProvider_NodeGroups_FullMethodName = "/clusterautoscaler.cloudprovider.v1.externalgrpc.CloudProvider/NodeGroups" @@ -118,8 +118,9 @@ func NewCloudProviderClient(cc grpc.ClientConnInterface) CloudProviderClient { } func (c *cloudProviderClient) NodeGroups(ctx context.Context, in *NodeGroupsRequest, opts ...grpc.CallOption) (*NodeGroupsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupsResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroups_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroups_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -127,8 +128,9 @@ func (c *cloudProviderClient) NodeGroups(ctx context.Context, in *NodeGroupsRequ } func (c *cloudProviderClient) NodeGroupForNode(ctx context.Context, in *NodeGroupForNodeRequest, opts ...grpc.CallOption) (*NodeGroupForNodeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupForNodeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupForNode_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupForNode_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -136,8 +138,9 @@ func (c *cloudProviderClient) NodeGroupForNode(ctx context.Context, in *NodeGrou } func (c *cloudProviderClient) PricingNodePrice(ctx context.Context, in *PricingNodePriceRequest, opts ...grpc.CallOption) (*PricingNodePriceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PricingNodePriceResponse) - err := c.cc.Invoke(ctx, CloudProvider_PricingNodePrice_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_PricingNodePrice_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -145,8 +148,9 @@ func (c *cloudProviderClient) PricingNodePrice(ctx context.Context, in *PricingN } func (c *cloudProviderClient) PricingPodPrice(ctx context.Context, in *PricingPodPriceRequest, opts ...grpc.CallOption) (*PricingPodPriceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PricingPodPriceResponse) - err := c.cc.Invoke(ctx, CloudProvider_PricingPodPrice_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_PricingPodPrice_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -154,8 +158,9 @@ func (c *cloudProviderClient) PricingPodPrice(ctx context.Context, in *PricingPo } func (c *cloudProviderClient) GPULabel(ctx context.Context, in *GPULabelRequest, opts ...grpc.CallOption) (*GPULabelResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GPULabelResponse) - err := c.cc.Invoke(ctx, CloudProvider_GPULabel_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_GPULabel_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -163,8 +168,9 @@ func (c *cloudProviderClient) GPULabel(ctx context.Context, in *GPULabelRequest, } func (c *cloudProviderClient) GetAvailableGPUTypes(ctx context.Context, in *GetAvailableGPUTypesRequest, opts ...grpc.CallOption) (*GetAvailableGPUTypesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetAvailableGPUTypesResponse) - err := c.cc.Invoke(ctx, CloudProvider_GetAvailableGPUTypes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_GetAvailableGPUTypes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -172,8 +178,9 @@ func (c *cloudProviderClient) GetAvailableGPUTypes(ctx context.Context, in *GetA } func (c *cloudProviderClient) Cleanup(ctx context.Context, in *CleanupRequest, opts ...grpc.CallOption) (*CleanupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CleanupResponse) - err := c.cc.Invoke(ctx, CloudProvider_Cleanup_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_Cleanup_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -181,8 +188,9 @@ func (c *cloudProviderClient) Cleanup(ctx context.Context, in *CleanupRequest, o } func (c *cloudProviderClient) Refresh(ctx context.Context, in *RefreshRequest, opts ...grpc.CallOption) (*RefreshResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RefreshResponse) - err := c.cc.Invoke(ctx, CloudProvider_Refresh_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_Refresh_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -190,8 +198,9 @@ func (c *cloudProviderClient) Refresh(ctx context.Context, in *RefreshRequest, o } func (c *cloudProviderClient) NodeGroupTargetSize(ctx context.Context, in *NodeGroupTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupTargetSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupTargetSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTargetSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTargetSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -199,8 +208,9 @@ func (c *cloudProviderClient) NodeGroupTargetSize(ctx context.Context, in *NodeG } func (c *cloudProviderClient) NodeGroupIncreaseSize(ctx context.Context, in *NodeGroupIncreaseSizeRequest, opts ...grpc.CallOption) (*NodeGroupIncreaseSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupIncreaseSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupIncreaseSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupIncreaseSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -208,8 +218,9 @@ func (c *cloudProviderClient) NodeGroupIncreaseSize(ctx context.Context, in *Nod } func (c *cloudProviderClient) NodeGroupDeleteNodes(ctx context.Context, in *NodeGroupDeleteNodesRequest, opts ...grpc.CallOption) (*NodeGroupDeleteNodesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupDeleteNodesResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDeleteNodes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDeleteNodes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -217,8 +228,9 @@ func (c *cloudProviderClient) NodeGroupDeleteNodes(ctx context.Context, in *Node } func (c *cloudProviderClient) NodeGroupDecreaseTargetSize(ctx context.Context, in *NodeGroupDecreaseTargetSizeRequest, opts ...grpc.CallOption) (*NodeGroupDecreaseTargetSizeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupDecreaseTargetSizeResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDecreaseTargetSize_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupDecreaseTargetSize_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -226,8 +238,9 @@ func (c *cloudProviderClient) NodeGroupDecreaseTargetSize(ctx context.Context, i } func (c *cloudProviderClient) NodeGroupNodes(ctx context.Context, in *NodeGroupNodesRequest, opts ...grpc.CallOption) (*NodeGroupNodesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupNodesResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupNodes_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupNodes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -235,8 +248,9 @@ func (c *cloudProviderClient) NodeGroupNodes(ctx context.Context, in *NodeGroupN } func (c *cloudProviderClient) NodeGroupTemplateNodeInfo(ctx context.Context, in *NodeGroupTemplateNodeInfoRequest, opts ...grpc.CallOption) (*NodeGroupTemplateNodeInfoResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupTemplateNodeInfoResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTemplateNodeInfo_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupTemplateNodeInfo_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -244,8 +258,9 @@ func (c *cloudProviderClient) NodeGroupTemplateNodeInfo(ctx context.Context, in } func (c *cloudProviderClient) NodeGroupGetOptions(ctx context.Context, in *NodeGroupAutoscalingOptionsRequest, opts ...grpc.CallOption) (*NodeGroupAutoscalingOptionsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(NodeGroupAutoscalingOptionsResponse) - err := c.cc.Invoke(ctx, CloudProvider_NodeGroupGetOptions_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, CloudProvider_NodeGroupGetOptions_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -254,7 +269,7 @@ func (c *cloudProviderClient) NodeGroupGetOptions(ctx context.Context, in *NodeG // CloudProviderServer is the server API for CloudProvider service. // All implementations must embed UnimplementedCloudProviderServer -// for forward compatibility +// for forward compatibility. type CloudProviderServer interface { // NodeGroups returns all node groups configured for this cloud provider. NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) @@ -311,9 +326,12 @@ type CloudProviderServer interface { mustEmbedUnimplementedCloudProviderServer() } -// UnimplementedCloudProviderServer must be embedded to have forward compatible implementations. -type UnimplementedCloudProviderServer struct { -} +// UnimplementedCloudProviderServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedCloudProviderServer struct{} func (UnimplementedCloudProviderServer) NodeGroups(context.Context, *NodeGroupsRequest) (*NodeGroupsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method NodeGroups not implemented") @@ -361,6 +379,7 @@ func (UnimplementedCloudProviderServer) NodeGroupGetOptions(context.Context, *No return nil, status.Errorf(codes.Unimplemented, "method NodeGroupGetOptions not implemented") } func (UnimplementedCloudProviderServer) mustEmbedUnimplementedCloudProviderServer() {} +func (UnimplementedCloudProviderServer) testEmbeddedByValue() {} // UnsafeCloudProviderServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to CloudProviderServer will @@ -370,6 +389,13 @@ type UnsafeCloudProviderServer interface { } func RegisterCloudProviderServer(s grpc.ServiceRegistrar, srv CloudProviderServer) { + // If the following call pancis, it indicates UnimplementedCloudProviderServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } s.RegisterService(&CloudProvider_ServiceDesc, srv) } diff --git a/cluster-autoscaler/cloudprovider/gce/gce_price_info.go b/cluster-autoscaler/cloudprovider/gce/gce_price_info.go index 003d3afde29..f71494d649d 100644 --- a/cluster-autoscaler/cloudprovider/gce/gce_price_info.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_price_info.go @@ -82,7 +82,7 @@ var ( "n2d": 0.027502, "n4": 0.030821, "t2d": 0.027502, - "z3": 0.04965, + "z3": 0.0496531, } predefinedMemoryPricePerHourPerGb = map[string]float64{ "a2": 0.004237, @@ -98,7 +98,7 @@ var ( "n2d": 0.003686, "n4": 0.004131, "t2d": 0.003686, - "z3": 0.00666, + "z3": 0.0066553, } predefinedPreemptibleDiscount = map[string]float64{ "a2": 0.009483 / 0.031611, @@ -114,7 +114,7 @@ var ( "n2d": 0.002773 / 0.027502, "n4": 0.007976 / 0.030821, "t2d": 0.006655 / 0.027502, - "z3": 0.01986 / 0.04965, + "z3": 0.0165 / 0.0496531, } customCpuPricePerHour = map[string]float64{ "e2": 0.022890, @@ -140,389 +140,421 @@ var ( // between the three machine types, the prices for e2-micro and e2-small // are artificially set to be higher than the price of e2-medium. instancePrices = map[string]float64{ - "a2-highgpu-1g": 3.673385, - "a2-highgpu-2g": 7.34677, - "a2-highgpu-4g": 14.69354, - "a2-highgpu-8g": 29.38708, - "a2-megagpu-16g": 55.739504, - "a2-ultragpu-1g": 5.0688, - "a2-ultragpu-2g": 10.1376, - "a2-ultragpu-4g": 20.2752, - "g2-standard-4": 0.76, - "g2-standard-8": 0.91, - "g2-standard-12": 1.06, - "g2-standard-16": 1.20, - "g2-standard-24": 2.11, - "g2-standard-32": 1.79, - "g2-standard-48": 4.23, - "g2-standard-96": 8.46, - "a2-ultragpu-8g": 40.5504, - "c2-standard-4": 0.2088, - "c2-standard-8": 0.4176, - "c2-standard-16": 0.8352, - "c2-standard-30": 1.5660, - "c2-standard-60": 3.1321, - "c2d-highcpu-2": 0.0750, - "c2d-highcpu-4": 0.1499, - "c2d-highcpu-8": 0.2998, - "c2d-highcpu-16": 0.5997, - "c2d-highcpu-32": 1.1994, - "c2d-highcpu-56": 2.0989, - "c2d-highcpu-112": 4.1979, - "c2d-highmem-2": 0.1225, - "c2d-highmem-4": 0.2449, - "c2d-highmem-8": 0.4899, - "c2d-highmem-16": 0.9798, - "c2d-highmem-32": 1.9595, - "c2d-highmem-56": 3.4292, - "c2d-highmem-112": 6.8583, - "c2d-standard-2": 0.0908, - "c2d-standard-4": 0.1816, - "c2d-standard-8": 0.3632, - "c2d-standard-16": 0.7264, - "c2d-standard-32": 1.4528, - "c2d-standard-56": 2.5423, - "c2d-standard-112": 5.0847, - "c3-standard-4": 0.20888, - "c3-standard-8": 0.41776, - "c3-standard-22": 1.14884, - "c3-standard-44": 2.29768, - "c3-standard-88": 4.59536, - "c3-standard-176": 9.19072, - "c3-highmem-4": 0.28184, - "c3-highmem-8": 0.56368, - "c3-highmem-22": 1.55012, - "c3-highmem-44": 3.10024, - "c3-highmem-88": 6.20048, - "c3-highmem-176": 12.40096, - "c3-highcpu-4": 0.1724, - "c3-highcpu-8": 0.3448, - "c3-highcpu-22": 0.9482, - "c3-highcpu-44": 1.8964, - "c3-highcpu-88": 3.7928, - "c3-highcpu-176": 7.5856, - "c3d-standard-4": 0.1816, - "c3d-standard-8": 0.3632, - "c3d-standard-16": 0.7264, - "c3d-standard-30": 1.362, - "c3d-standard-60": 2.724, - "c3d-standard-90": 4.086, - "c3d-standard-180": 8.172, - "c3d-standard-360": 16.344, - "c3d-highmem-4": 0.24496, - "c3d-highmem-8": 0.48992, - "c3d-highmem-16": 0.97984, - "c3d-highmem-30": 1.8372, - "c3d-highmem-60": 3.6744, - "c3d-highmem-90": 5.5116, - "c3d-highmem-180": 11.0232, - "c3d-highmem-360": 22.0464, - "c3d-highcpu-4": 0.14992, - "c3d-highcpu-8": 0.29984, - "c3d-highcpu-16": 0.59968, - "c3d-highcpu-30": 1.1244, - "c3d-highcpu-60": 2.2488, - "c3d-highcpu-90": 3.3732, - "c3d-highcpu-180": 6.7464, - "c3d-highcpu-360": 13.4928, - "e2-highcpu-2": 0.04947, - "e2-highcpu-4": 0.09894, - "e2-highcpu-8": 0.19788, - "e2-highcpu-16": 0.39576, - "e2-highcpu-32": 0.79149, - "e2-highmem-2": 0.09040, - "e2-highmem-4": 0.18080, - "e2-highmem-8": 0.36160, - "e2-highmem-16": 0.72320, - "e2-medium": 0.03351, - "e2-micro": 0.03353, // Should be 0.00838. Set to be > e2-medium. - "e2-small": 0.03352, // Should be 0.01675. Set to be > e2-medium. - "e2-standard-2": 0.06701, - "e2-standard-4": 0.13402, - "e2-standard-8": 0.26805, - "e2-standard-16": 0.53609, - "e2-standard-32": 1.07210, - "f1-micro": 0.0076, - "g1-small": 0.0257, - "m1-megamem-96": 10.6740, - "m1-ultramem-40": 6.3039, - "m1-ultramem-80": 12.6078, - "m1-ultramem-160": 25.2156, - "m2-ultramem-208": 42.186, - "m2-ultramem-416": 84.371, - "m2-megamem-416": 50.372, - "n1-highcpu-2": 0.0709, - "n1-highcpu-4": 0.1418, - "n1-highcpu-8": 0.2836, - "n1-highcpu-16": 0.5672, - "n1-highcpu-32": 1.1344, - "n1-highcpu-64": 2.2688, - "n1-highcpu-96": 3.402, - "n1-highmem-2": 0.1184, - "n1-highmem-4": 0.2368, - "n1-highmem-8": 0.4736, - "n1-highmem-16": 0.9472, - "n1-highmem-32": 1.8944, - "n1-highmem-64": 3.7888, - "n1-highmem-96": 5.6832, - "n1-standard-1": 0.0475, - "n1-standard-2": 0.0950, - "n1-standard-4": 0.1900, - "n1-standard-8": 0.3800, - "n1-standard-16": 0.7600, - "n1-standard-32": 1.5200, - "n1-standard-64": 3.0400, - "n1-standard-96": 4.5600, - "n2-highcpu-2": 0.0717, - "n2-highcpu-4": 0.1434, - "n2-highcpu-8": 0.2868, - "n2-highcpu-16": 0.5736, - "n2-highcpu-32": 1.1471, - "n2-highcpu-48": 1.7207, - "n2-highcpu-64": 2.2943, - "n2-highcpu-80": 2.8678, - "n2-highcpu-96": 3.4414, - "n2-highcpu-128": 4.5886, - "n2-highmem-2": 0.1310, - "n2-highmem-4": 0.2620, - "n2-highmem-8": 0.5241, - "n2-highmem-16": 1.0481, - "n2-highmem-32": 2.0962, - "n2-highmem-48": 3.1443, - "n2-highmem-64": 4.1924, - "n2-highmem-80": 5.2406, - "n2-highmem-96": 6.2886, - "n2-highmem-128": 7.7069, - "n2-standard-2": 0.0971, - "n2-standard-4": 0.1942, - "n2-standard-8": 0.3885, - "n2-standard-16": 0.7769, - "n2-standard-32": 1.5539, - "n2-standard-48": 2.3308, - "n2-standard-64": 3.1078, - "n2-standard-80": 3.8847, - "n2-standard-96": 4.6616, - "n2-standard-128": 6.2156, - "n2d-highcpu-2": 0.0624, - "n2d-highcpu-4": 0.1248, - "n2d-highcpu-8": 0.2495, - "n2d-highcpu-16": 0.4990, - "n2d-highcpu-32": 0.9980, - "n2d-highcpu-48": 1.4970, - "n2d-highcpu-64": 1.9960, - "n2d-highcpu-80": 2.4950, - "n2d-highcpu-96": 2.9940, - "n2d-highcpu-128": 3.9920, - "n2d-highcpu-224": 6.9861, - "n2d-highmem-2": 0.1140, - "n2d-highmem-4": 0.2280, - "n2d-highmem-8": 0.4559, - "n2d-highmem-16": 0.9119, - "n2d-highmem-32": 1.8237, - "n2d-highmem-48": 2.7356, - "n2d-highmem-64": 3.6474, - "n2d-highmem-80": 4.5593, - "n2d-highmem-96": 5.4711, - "n2d-standard-2": 0.0845, - "n2d-standard-4": 0.1690, - "n2d-standard-8": 0.3380, - "n2d-standard-16": 0.6759, - "n2d-standard-32": 1.3519, - "n2d-standard-48": 2.0278, - "n2d-standard-64": 2.7038, - "n2d-standard-80": 3.3797, - "n2d-standard-96": 4.0556, - "n2d-standard-128": 5.4075, - "n2d-standard-224": 9.4632, - "t2d-standard-1": 0.0422, - "t2d-standard-2": 0.0845, - "t2d-standard-4": 0.1690, - "t2d-standard-8": 0.3380, - "t2d-standard-16": 0.6759, - "t2d-standard-32": 1.3519, - "t2d-standard-48": 2.0278, - "t2d-standard-60": 2.5348, - "z3-highmem-88": 13.0, - "z3-highmem-176": 22.05, + "a2-highgpu-1g": 3.673385, + "a2-highgpu-2g": 7.34677, + "a2-highgpu-4g": 14.69354, + "a2-highgpu-8g": 29.38708, + "a2-megagpu-16g": 55.739504, + "a2-ultragpu-1g": 5.0688, + "a2-ultragpu-2g": 10.1376, + "a2-ultragpu-4g": 20.2752, + "g2-standard-4": 0.76, + "g2-standard-8": 0.91, + "g2-standard-12": 1.06, + "g2-standard-16": 1.20, + "g2-standard-24": 2.11, + "g2-standard-32": 1.79, + "g2-standard-48": 4.23, + "g2-standard-96": 8.46, + "a2-ultragpu-8g": 40.5504, + "c2-standard-4": 0.2088, + "c2-standard-8": 0.4176, + "c2-standard-16": 0.8352, + "c2-standard-30": 1.5660, + "c2-standard-60": 3.1321, + "c2d-highcpu-2": 0.0750, + "c2d-highcpu-4": 0.1499, + "c2d-highcpu-8": 0.2998, + "c2d-highcpu-16": 0.5997, + "c2d-highcpu-32": 1.1994, + "c2d-highcpu-56": 2.0989, + "c2d-highcpu-112": 4.1979, + "c2d-highmem-2": 0.1225, + "c2d-highmem-4": 0.2449, + "c2d-highmem-8": 0.4899, + "c2d-highmem-16": 0.9798, + "c2d-highmem-32": 1.9595, + "c2d-highmem-56": 3.4292, + "c2d-highmem-112": 6.8583, + "c2d-standard-2": 0.0908, + "c2d-standard-4": 0.1816, + "c2d-standard-8": 0.3632, + "c2d-standard-16": 0.7264, + "c2d-standard-32": 1.4528, + "c2d-standard-56": 2.5423, + "c2d-standard-112": 5.0847, + "c3-standard-4": 0.20888, + "c3-standard-8": 0.41776, + "c3-standard-22": 1.14884, + "c3-standard-44": 2.29768, + "c3-standard-88": 4.59536, + "c3-standard-176": 9.19072, + "c3-highmem-4": 0.28184, + "c3-highmem-8": 0.56368, + "c3-highmem-22": 1.55012, + "c3-highmem-44": 3.10024, + "c3-highmem-88": 6.20048, + "c3-highmem-176": 12.40096, + "c3-highcpu-4": 0.1724, + "c3-highcpu-8": 0.3448, + "c3-highcpu-22": 0.9482, + "c3-highcpu-44": 1.8964, + "c3-highcpu-88": 3.7928, + "c3-highcpu-176": 7.5856, + "c3d-standard-4": 0.1816, + "c3d-standard-8": 0.3632, + "c3d-standard-16": 0.7264, + "c3d-standard-30": 1.362, + "c3d-standard-60": 2.724, + "c3d-standard-90": 4.086, + "c3d-standard-180": 8.172, + "c3d-standard-360": 16.344, + "c3d-highmem-4": 0.24496, + "c3d-highmem-8": 0.48992, + "c3d-highmem-16": 0.97984, + "c3d-highmem-30": 1.8372, + "c3d-highmem-60": 3.6744, + "c3d-highmem-90": 5.5116, + "c3d-highmem-180": 11.0232, + "c3d-highmem-360": 22.0464, + "c3d-highcpu-4": 0.14992, + "c3d-highcpu-8": 0.29984, + "c3d-highcpu-16": 0.59968, + "c3d-highcpu-30": 1.1244, + "c3d-highcpu-60": 2.2488, + "c3d-highcpu-90": 3.3732, + "c3d-highcpu-180": 6.7464, + "c3d-highcpu-360": 13.4928, + "e2-highcpu-2": 0.04947, + "e2-highcpu-4": 0.09894, + "e2-highcpu-8": 0.19788, + "e2-highcpu-16": 0.39576, + "e2-highcpu-32": 0.79149, + "e2-highmem-2": 0.09040, + "e2-highmem-4": 0.18080, + "e2-highmem-8": 0.36160, + "e2-highmem-16": 0.72320, + "e2-medium": 0.03351, + "e2-micro": 0.03353, // Should be 0.00838. Set to be > e2-medium. + "e2-small": 0.03352, // Should be 0.01675. Set to be > e2-medium. + "e2-standard-2": 0.06701, + "e2-standard-4": 0.13402, + "e2-standard-8": 0.26805, + "e2-standard-16": 0.53609, + "e2-standard-32": 1.07210, + "f1-micro": 0.0076, + "g1-small": 0.0257, + "m1-megamem-96": 10.6740, + "m1-ultramem-40": 6.3039, + "m1-ultramem-80": 12.6078, + "m1-ultramem-160": 25.2156, + "m2-ultramem-208": 42.186, + "m2-ultramem-416": 84.371, + "m2-megamem-416": 50.372, + "n1-highcpu-2": 0.0709, + "n1-highcpu-4": 0.1418, + "n1-highcpu-8": 0.2836, + "n1-highcpu-16": 0.5672, + "n1-highcpu-32": 1.1344, + "n1-highcpu-64": 2.2688, + "n1-highcpu-96": 3.402, + "n1-highmem-2": 0.1184, + "n1-highmem-4": 0.2368, + "n1-highmem-8": 0.4736, + "n1-highmem-16": 0.9472, + "n1-highmem-32": 1.8944, + "n1-highmem-64": 3.7888, + "n1-highmem-96": 5.6832, + "n1-standard-1": 0.0475, + "n1-standard-2": 0.0950, + "n1-standard-4": 0.1900, + "n1-standard-8": 0.3800, + "n1-standard-16": 0.7600, + "n1-standard-32": 1.5200, + "n1-standard-64": 3.0400, + "n1-standard-96": 4.5600, + "n2-highcpu-2": 0.0717, + "n2-highcpu-4": 0.1434, + "n2-highcpu-8": 0.2868, + "n2-highcpu-16": 0.5736, + "n2-highcpu-32": 1.1471, + "n2-highcpu-48": 1.7207, + "n2-highcpu-64": 2.2943, + "n2-highcpu-80": 2.8678, + "n2-highcpu-96": 3.4414, + "n2-highcpu-128": 4.5886, + "n2-highmem-2": 0.1310, + "n2-highmem-4": 0.2620, + "n2-highmem-8": 0.5241, + "n2-highmem-16": 1.0481, + "n2-highmem-32": 2.0962, + "n2-highmem-48": 3.1443, + "n2-highmem-64": 4.1924, + "n2-highmem-80": 5.2406, + "n2-highmem-96": 6.2886, + "n2-highmem-128": 7.7069, + "n2-standard-2": 0.0971, + "n2-standard-4": 0.1942, + "n2-standard-8": 0.3885, + "n2-standard-16": 0.7769, + "n2-standard-32": 1.5539, + "n2-standard-48": 2.3308, + "n2-standard-64": 3.1078, + "n2-standard-80": 3.8847, + "n2-standard-96": 4.6616, + "n2-standard-128": 6.2156, + "n2d-highcpu-2": 0.0624, + "n2d-highcpu-4": 0.1248, + "n2d-highcpu-8": 0.2495, + "n2d-highcpu-16": 0.4990, + "n2d-highcpu-32": 0.9980, + "n2d-highcpu-48": 1.4970, + "n2d-highcpu-64": 1.9960, + "n2d-highcpu-80": 2.4950, + "n2d-highcpu-96": 2.9940, + "n2d-highcpu-128": 3.9920, + "n2d-highcpu-224": 6.9861, + "n2d-highmem-2": 0.1140, + "n2d-highmem-4": 0.2280, + "n2d-highmem-8": 0.4559, + "n2d-highmem-16": 0.9119, + "n2d-highmem-32": 1.8237, + "n2d-highmem-48": 2.7356, + "n2d-highmem-64": 3.6474, + "n2d-highmem-80": 4.5593, + "n2d-highmem-96": 5.4711, + "n2d-standard-2": 0.0845, + "n2d-standard-4": 0.1690, + "n2d-standard-8": 0.3380, + "n2d-standard-16": 0.6759, + "n2d-standard-32": 1.3519, + "n2d-standard-48": 2.0278, + "n2d-standard-64": 2.7038, + "n2d-standard-80": 3.3797, + "n2d-standard-96": 4.0556, + "n2d-standard-128": 5.4075, + "n2d-standard-224": 9.4632, + "t2d-standard-1": 0.0422, + "t2d-standard-2": 0.0845, + "t2d-standard-4": 0.1690, + "t2d-standard-8": 0.3380, + "t2d-standard-16": 0.6759, + "t2d-standard-32": 1.3519, + "t2d-standard-48": 2.0278, + "t2d-standard-60": 2.5348, + "z3-highmem-176": 21.980576, + "z3-highmem-8": 1.145745, + "z3-highmem-14": 2.085698, + "z3-highmem-22": 3.231443, + "z3-highmem-30": 4.377188, + "z3-highmem-36": 5.317141, + "z3-highmem-44": 6.462886, + "z3-highmem-8-highlssd": 1.145745, + "z3-highmem-16-highlssd": 2.291489, + "z3-highmem-22-highlssd": 3.231443, + "z3-highmem-32-highlssd": 4.582979, + "z3-highmem-44-highlssd": 6.462886, + "z3-highmem-88-highlssd": 12.925772, + "z3-highmem-14-standardlssd": 1.763118, + "z3-highmem-22-standardlssd": 2.908862, + "z3-highmem-44-standardlssd": 5.495144, + "z3-highmem-88-standardlssd": 10.990288, + "z3-highmem-176-standardlssd": 21.980576, } preemptiblePrices = map[string]float64{ - "a2-highgpu-1g": 1.102016, - "a2-highgpu-2g": 2.204031, - "a2-highgpu-4g": 4.408062, - "a2-highgpu-8g": 8.816124, - "a2-megagpu-16g": 16.721851, - "a2-ultragpu-1g": 1.6, - "a2-ultragpu-2g": 3.2, - "a2-ultragpu-4g": 6.4, - "a2-ultragpu-8g": 12.8, - "g2-standard-4": 0.23, - "g2-standard-8": 0.27, - "g2-standard-12": 0.32, - "g2-standard-16": 0.36, - "g2-standard-24": 0.63, - "g2-standard-32": 0.54, - "g2-standard-48": 1.27, - "g2-standard-96": 2.54, - "c2-standard-4": 0.0505, - "c2-standard-8": 0.1011, - "c2-standard-16": 0.2021, - "c2-standard-30": 0.3790, - "c2-standard-60": 0.7579, - "c2d-highcpu-2": 0.0181, - "c2d-highcpu-4": 0.0363, - "c2d-highcpu-8": 0.0726, - "c2d-highcpu-16": 0.1451, - "c2d-highcpu-32": 0.2902, - "c2d-highcpu-56": 0.5079, - "c2d-highcpu-112": 1.0158, - "c2d-highmem-2": 0.0296, - "c2d-highmem-4": 0.0593, - "c2d-highmem-8": 0.1185, - "c2d-highmem-16": 0.2371, - "c2d-highmem-32": 0.4742, - "c2d-highmem-56": 0.8298, - "c2d-highmem-112": 1.6596, - "c2d-standard-2": 0.0220, - "c2d-standard-4": 0.0439, - "c2d-standard-8": 0.0879, - "c2d-standard-16": 0.1758, - "c2d-standard-32": 0.3516, - "c2d-standard-56": 0.6152, - "c2d-standard-112": 1.2304, - "c3-standard-4": 0.018952, - "c3-standard-8": 0.037904, - "c3-standard-22": 0.104236, - "c3-standard-44": 0.208472, - "c3-standard-88": 0.416944, - "c3-standard-176": 0.833888, - "c3-highmem-4": 0.02556, - "c3-highmem-8": 0.05112, - "c3-highmem-22": 0.14058, - "c3-highmem-44": 0.28116, - "c3-highmem-88": 0.56232, - "c3-highmem-176": 1.12464, - "c3-highcpu-4": 0.015648, - "c3-highcpu-8": 0.031296, - "c3-highcpu-22": 0.086064, - "c3-highcpu-44": 0.172128, - "c3-highcpu-88": 0.344256, - "c3-highcpu-176": 0.688512, - "e2-highcpu-2": 0.01484, - "e2-highcpu-4": 0.02968, - "e2-highcpu-8": 0.05936, - "e2-highcpu-16": 0.11873, - "e2-highcpu-32": 0.23744, - "e2-highmem-2": 0.02712, - "e2-highmem-4": 0.05424, - "e2-highmem-8": 0.10848, - "e2-highmem-16": 0.21696, - "e2-medium": 0.01005, - "e2-micro": 0.01007, // Should be 0.00251. Set to be > e2-medium. - "e2-small": 0.01006, // Should be 0.00503. Set to be > e2-medium. - "e2-standard-2": 0.02010, - "e2-standard-4": 0.04021, - "e2-standard-8": 0.08041, - "e2-standard-16": 0.16083, - "e2-standard-32": 0.32163, - "f1-micro": 0.0035, - "g1-small": 0.0070, - "m1-megamem-96": 2.2600, - "m1-ultramem-40": 1.3311, - "m1-ultramem-80": 2.6622, - "m1-ultramem-160": 5.3244, - "n1-highcpu-2": 0.0150, - "n1-highcpu-4": 0.0300, - "n1-highcpu-8": 0.0600, - "n1-highcpu-16": 0.1200, - "n1-highcpu-32": 0.2400, - "n1-highcpu-64": 0.4800, - "n1-highcpu-96": 0.7200, - "n1-highmem-2": 0.0250, - "n1-highmem-4": 0.0500, - "n1-highmem-8": 0.1000, - "n1-highmem-16": 0.2000, - "n1-highmem-32": 0.4000, - "n1-highmem-64": 0.8000, - "n1-highmem-96": 1.2000, - "n1-standard-1": 0.0100, - "n1-standard-2": 0.0200, - "n1-standard-4": 0.0400, - "n1-standard-8": 0.0800, - "n1-standard-16": 0.1600, - "n1-standard-32": 0.3200, - "n1-standard-64": 0.6400, - "n1-standard-96": 0.9600, - "n2-highcpu-2": 0.0173, - "n2-highcpu-4": 0.0347, - "n2-highcpu-8": 0.0694, - "n2-highcpu-16": 0.1388, - "n2-highcpu-32": 0.2776, - "n2-highcpu-48": 0.4164, - "n2-highcpu-64": 0.5552, - "n2-highcpu-80": 0.6940, - "n2-highcpu-96": 0.8328, - "n2-highcpu-128": 1.1104, - "n2-highmem-2": 0.0317, - "n2-highmem-4": 0.0634, - "n2-highmem-8": 0.1268, - "n2-highmem-16": 0.2536, - "n2-highmem-32": 0.5073, - "n2-highmem-48": 0.7609, - "n2-highmem-64": 1.0145, - "n2-highmem-80": 1.2681, - "n2-highmem-96": 1.5218, - "n2-highmem-128": 1.8691, - "n2-standard-2": 0.0235, - "n2-standard-4": 0.0470, - "n2-standard-8": 0.0940, - "n2-standard-16": 0.1880, - "n2-standard-32": 0.3760, - "n2-standard-48": 0.5640, - "n2-standard-64": 0.7520, - "n2-standard-80": 0.9400, - "n2-standard-96": 1.128, - "n2-standard-128": 1.504, - "n2d-highcpu-2": 0.00629, - "n2d-highcpu-4": 0.01258, - "n2d-highcpu-8": 0.02516, - "n2d-highcpu-16": 0.05032, - "n2d-highcpu-32": 0.10064, - "n2d-highcpu-48": 0.15096, - "n2d-highcpu-64": 0.20128, - "n2d-highcpu-80": 0.2516, - "n2d-highcpu-96": 0.30192, - "n2d-highcpu-128": 0.40256, - "n2d-highcpu-224": 0.70448, - "n2d-highmem-2": 0.011498, - "n2d-highmem-4": 0.022996, - "n2d-highmem-8": 0.045992, - "n2d-highmem-16": 0.091984, - "n2d-highmem-32": 0.183968, - "n2d-highmem-48": 0.275952, - "n2d-highmem-64": 0.367936, - "n2d-highmem-80": 0.45992, - "n2d-highmem-96": 0.551904, - "n2d-standard-2": 0.008522, - "n2d-standard-4": 0.017044, - "n2d-standard-8": 0.034088, - "n2d-standard-16": 0.068176, - "n2d-standard-32": 0.136352, - "n2d-standard-48": 0.204528, - "n2d-standard-64": 0.272704, - "n2d-standard-80": 0.34088, - "n2d-standard-96": 0.409056, - "n2d-standard-128": 0.545408, - "n2d-standard-224": 0.954464, - "t2d-standard-1": 0.0102, - "t2d-standard-2": 0.0204, - "t2d-standard-4": 0.0409, - "t2d-standard-8": 0.0818, - "t2d-standard-16": 0.1636, - "t2d-standard-32": 0.3271, - "t2d-standard-48": 0.4907, - "t2d-standard-60": 0.6134, - "z3-highmem-88": 5.2, - "z3-highmem-176": 8.82, + "a2-highgpu-1g": 1.102016, + "a2-highgpu-2g": 2.204031, + "a2-highgpu-4g": 4.408062, + "a2-highgpu-8g": 8.816124, + "a2-megagpu-16g": 16.721851, + "a2-ultragpu-1g": 1.6, + "a2-ultragpu-2g": 3.2, + "a2-ultragpu-4g": 6.4, + "a2-ultragpu-8g": 12.8, + "g2-standard-4": 0.23, + "g2-standard-8": 0.27, + "g2-standard-12": 0.32, + "g2-standard-16": 0.36, + "g2-standard-24": 0.63, + "g2-standard-32": 0.54, + "g2-standard-48": 1.27, + "g2-standard-96": 2.54, + "c2-standard-4": 0.0505, + "c2-standard-8": 0.1011, + "c2-standard-16": 0.2021, + "c2-standard-30": 0.3790, + "c2-standard-60": 0.7579, + "c2d-highcpu-2": 0.0181, + "c2d-highcpu-4": 0.0363, + "c2d-highcpu-8": 0.0726, + "c2d-highcpu-16": 0.1451, + "c2d-highcpu-32": 0.2902, + "c2d-highcpu-56": 0.5079, + "c2d-highcpu-112": 1.0158, + "c2d-highmem-2": 0.0296, + "c2d-highmem-4": 0.0593, + "c2d-highmem-8": 0.1185, + "c2d-highmem-16": 0.2371, + "c2d-highmem-32": 0.4742, + "c2d-highmem-56": 0.8298, + "c2d-highmem-112": 1.6596, + "c2d-standard-2": 0.0220, + "c2d-standard-4": 0.0439, + "c2d-standard-8": 0.0879, + "c2d-standard-16": 0.1758, + "c2d-standard-32": 0.3516, + "c2d-standard-56": 0.6152, + "c2d-standard-112": 1.2304, + "c3-standard-4": 0.018952, + "c3-standard-8": 0.037904, + "c3-standard-22": 0.104236, + "c3-standard-44": 0.208472, + "c3-standard-88": 0.416944, + "c3-standard-176": 0.833888, + "c3-highmem-4": 0.02556, + "c3-highmem-8": 0.05112, + "c3-highmem-22": 0.14058, + "c3-highmem-44": 0.28116, + "c3-highmem-88": 0.56232, + "c3-highmem-176": 1.12464, + "c3-highcpu-4": 0.015648, + "c3-highcpu-8": 0.031296, + "c3-highcpu-22": 0.086064, + "c3-highcpu-44": 0.172128, + "c3-highcpu-88": 0.344256, + "c3-highcpu-176": 0.688512, + "e2-highcpu-2": 0.01484, + "e2-highcpu-4": 0.02968, + "e2-highcpu-8": 0.05936, + "e2-highcpu-16": 0.11873, + "e2-highcpu-32": 0.23744, + "e2-highmem-2": 0.02712, + "e2-highmem-4": 0.05424, + "e2-highmem-8": 0.10848, + "e2-highmem-16": 0.21696, + "e2-medium": 0.01005, + "e2-micro": 0.01007, // Should be 0.00251. Set to be > e2-medium. + "e2-small": 0.01006, // Should be 0.00503. Set to be > e2-medium. + "e2-standard-2": 0.02010, + "e2-standard-4": 0.04021, + "e2-standard-8": 0.08041, + "e2-standard-16": 0.16083, + "e2-standard-32": 0.32163, + "f1-micro": 0.0035, + "g1-small": 0.0070, + "m1-megamem-96": 2.2600, + "m1-ultramem-40": 1.3311, + "m1-ultramem-80": 2.6622, + "m1-ultramem-160": 5.3244, + "n1-highcpu-2": 0.0150, + "n1-highcpu-4": 0.0300, + "n1-highcpu-8": 0.0600, + "n1-highcpu-16": 0.1200, + "n1-highcpu-32": 0.2400, + "n1-highcpu-64": 0.4800, + "n1-highcpu-96": 0.7200, + "n1-highmem-2": 0.0250, + "n1-highmem-4": 0.0500, + "n1-highmem-8": 0.1000, + "n1-highmem-16": 0.2000, + "n1-highmem-32": 0.4000, + "n1-highmem-64": 0.8000, + "n1-highmem-96": 1.2000, + "n1-standard-1": 0.0100, + "n1-standard-2": 0.0200, + "n1-standard-4": 0.0400, + "n1-standard-8": 0.0800, + "n1-standard-16": 0.1600, + "n1-standard-32": 0.3200, + "n1-standard-64": 0.6400, + "n1-standard-96": 0.9600, + "n2-highcpu-2": 0.0173, + "n2-highcpu-4": 0.0347, + "n2-highcpu-8": 0.0694, + "n2-highcpu-16": 0.1388, + "n2-highcpu-32": 0.2776, + "n2-highcpu-48": 0.4164, + "n2-highcpu-64": 0.5552, + "n2-highcpu-80": 0.6940, + "n2-highcpu-96": 0.8328, + "n2-highcpu-128": 1.1104, + "n2-highmem-2": 0.0317, + "n2-highmem-4": 0.0634, + "n2-highmem-8": 0.1268, + "n2-highmem-16": 0.2536, + "n2-highmem-32": 0.5073, + "n2-highmem-48": 0.7609, + "n2-highmem-64": 1.0145, + "n2-highmem-80": 1.2681, + "n2-highmem-96": 1.5218, + "n2-highmem-128": 1.8691, + "n2-standard-2": 0.0235, + "n2-standard-4": 0.0470, + "n2-standard-8": 0.0940, + "n2-standard-16": 0.1880, + "n2-standard-32": 0.3760, + "n2-standard-48": 0.5640, + "n2-standard-64": 0.7520, + "n2-standard-80": 0.9400, + "n2-standard-96": 1.128, + "n2-standard-128": 1.504, + "n2d-highcpu-2": 0.00629, + "n2d-highcpu-4": 0.01258, + "n2d-highcpu-8": 0.02516, + "n2d-highcpu-16": 0.05032, + "n2d-highcpu-32": 0.10064, + "n2d-highcpu-48": 0.15096, + "n2d-highcpu-64": 0.20128, + "n2d-highcpu-80": 0.2516, + "n2d-highcpu-96": 0.30192, + "n2d-highcpu-128": 0.40256, + "n2d-highcpu-224": 0.70448, + "n2d-highmem-2": 0.011498, + "n2d-highmem-4": 0.022996, + "n2d-highmem-8": 0.045992, + "n2d-highmem-16": 0.091984, + "n2d-highmem-32": 0.183968, + "n2d-highmem-48": 0.275952, + "n2d-highmem-64": 0.367936, + "n2d-highmem-80": 0.45992, + "n2d-highmem-96": 0.551904, + "n2d-standard-2": 0.008522, + "n2d-standard-4": 0.017044, + "n2d-standard-8": 0.034088, + "n2d-standard-16": 0.068176, + "n2d-standard-32": 0.136352, + "n2d-standard-48": 0.204528, + "n2d-standard-64": 0.272704, + "n2d-standard-80": 0.34088, + "n2d-standard-96": 0.409056, + "n2d-standard-128": 0.545408, + "n2d-standard-224": 0.954464, + "t2d-standard-1": 0.0102, + "t2d-standard-2": 0.0204, + "t2d-standard-4": 0.0409, + "t2d-standard-8": 0.0818, + "t2d-standard-16": 0.1636, + "t2d-standard-32": 0.3271, + "t2d-standard-48": 0.4907, + "t2d-standard-60": 0.6134, + "z3-highmem-176": 7.568291, + "z3-highmem-8": 0.402664, + "z3-highmem-14": 0.736921, + "z3-highmem-22": 1.139585, + "z3-highmem-30": 1.542249, + "z3-highmem-36": 1.876505, + "z3-highmem-44": 2.279170, + "z3-highmem-8-highlssd": 0.402664, + "z3-highmem-16-highlssd": 0.805329, + "z3-highmem-22-highlssd": 1.139585, + "z3-highmem-32-highlssd": 1.610657, + "z3-highmem-44-highlssd": 2.279170, + "z3-highmem-88-highlssd": 4.558339, + "z3-highmem-14-standardlssd": 0.607888, + "z3-highmem-22-standardlssd": 1.010553, + "z3-highmem-44-standardlssd": 1.892073, + "z3-highmem-88-standardlssd": 3.784146, + "z3-highmem-176-standardlssd": 7.568291, } gpuPrices = map[string]float64{ "nvidia-tesla-t4": 0.35, diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index d9d139763a0..cedc37e71ff 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -276,6 +276,8 @@ type AutoscalingOptions struct { // MinReplicaCount controls the minimum number of replicas that a replica set or replication controller should have // to allow their pods deletion in scale down MinReplicaCount int + // BspDisruptionTimeout is the timeout after which CA will evict non-pdb-assigned blocking system pods + BspDisruptionTimeout time.Duration // NodeDeleteDelayAfterTaint is the duration to wait before deleting a node after tainting it NodeDeleteDelayAfterTaint time.Duration // NodeGroupSetRatio is a collection of ratios used by CA used to make scaling decisions. diff --git a/cluster-autoscaler/config/flags/flags.go b/cluster-autoscaler/config/flags/flags.go index 0838a8a3652..11533a08221 100644 --- a/cluster-autoscaler/config/flags/flags.go +++ b/cluster-autoscaler/config/flags/flags.go @@ -77,7 +77,7 @@ var ( scaleDownDelayTypeLocal = flag.Bool("scale-down-delay-type-local", false, "Should --scale-down-delay-after-* flags be applied locally per nodegroup or globally across all nodegroups") scaleDownDelayAfterDelete = flag.Duration("scale-down-delay-after-delete", 0, - "How long after node deletion that scale down evaluation resumes, defaults to scanInterval") + "How long after node deletion that scale down evaluation resumes") scaleDownDelayAfterFailure = flag.Duration("scale-down-delay-after-failure", config.DefaultScaleDownDelayAfterFailure, "How long after scale down failure that scale down evaluation resumes") scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", config.DefaultScaleDownUnneededTime, @@ -193,10 +193,11 @@ var ( recordDuplicatedEvents = flag.Bool("record-duplicated-events", false, "enable duplication of similar events within a 5 minute window.") maxNodesPerScaleUp = flag.Int("max-nodes-per-scaleup", 1000, "Max nodes added in a single scale-up. This is intended strictly for optimizing CA algorithm latency and not a tool to rate-limit scale-up throughput.") maxNodeGroupBinpackingDuration = flag.Duration("max-nodegroup-binpacking-duration", 10*time.Second, "Maximum time that will be spent in binpacking simulation for each NodeGroup.") - skipNodesWithSystemPods = flag.Bool("skip-nodes-with-system-pods", true, "If true cluster autoscaler will never delete nodes with pods from kube-system (except for DaemonSet or mirror pods)") + skipNodesWithSystemPods = flag.Bool("skip-nodes-with-system-pods", true, "If true cluster autoscaler will wait for --blocking-system-pod-distruption-timeout before deleting nodes with pods from kube-system (except for DaemonSet or mirror pods)") skipNodesWithLocalStorage = flag.Bool("skip-nodes-with-local-storage", true, "If true cluster autoscaler will never delete nodes with pods with local storage, e.g. EmptyDir or HostPath") skipNodesWithCustomControllerPods = flag.Bool("skip-nodes-with-custom-controller-pods", true, "If true cluster autoscaler will never delete nodes with pods owned by custom controllers") minReplicaCount = flag.Int("min-replica-count", 0, "Minimum number or replicas that a replica set or replication controller should have to allow their pods deletion in scale down") + bspDisruptionTimeout = flag.Duration("blocking-system-pod-distruption-timeout", time.Hour, "The timeout after which CA will evict non-pdb-assigned blocking system pods, applicable only when --skip-nodes-with-system-pods is set to true") nodeDeleteDelayAfterTaint = flag.Duration("node-delete-delay-after-taint", 5*time.Second, "How long to wait before deleting a node after tainting it") scaleDownSimulationTimeout = flag.Duration("scale-down-simulation-timeout", 30*time.Second, "How long should we run scale down simulation.") maxCapacityMemoryDifferenceRatio = flag.Float64("memory-difference-ratio", config.DefaultMaxCapacityMemoryDifferenceRatio, "Maximum difference in memory capacity between two similar node groups to be considered for balancing. Value is a ratio of the smaller node group's memory capacity.") @@ -370,6 +371,7 @@ func createAutoscalingOptions() config.AutoscalingOptions { SkipNodesWithSystemPods: *skipNodesWithSystemPods, SkipNodesWithLocalStorage: *skipNodesWithLocalStorage, MinReplicaCount: *minReplicaCount, + BspDisruptionTimeout: *bspDisruptionTimeout, NodeDeleteDelayAfterTaint: *nodeDeleteDelayAfterTaint, ScaleDownSimulationTimeout: *scaleDownSimulationTimeout, SkipNodesWithCustomControllerPods: *skipNodesWithCustomControllerPods, diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go index e0dd551172c..3cae232ad74 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go @@ -155,9 +155,17 @@ func (s *AsyncNodeGroupInitializer) InitializeNodeGroup(result nodegroups.AsyncN return } klog.Infof("Initial scale-up succeeded. Scale ups: %v", scaleUpInfos) + s.emitScaleUpStatus(&status.ScaleUpStatus{ + Result: status.ScaleUpSuccessful, + ScaleUpInfos: scaleUpInfos, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{result.CreationResult}, + PodsTriggeredScaleUp: s.triggeringPods, + }, nil) } func (s *AsyncNodeGroupInitializer) emitScaleUpStatus(scaleUpStatus *status.ScaleUpStatus, err errors.AutoscalerError) { - status.UpdateScaleUpError(scaleUpStatus, err) + if err != nil { + status.UpdateScaleUpError(scaleUpStatus, err) + } s.scaleUpStatusProcessor.Process(s.context, scaleUpStatus) } diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go index c566328f985..088fc0dd041 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups/asyncnodegroups" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" "k8s.io/autoscaler/cluster-autoscaler/processors/status" processorstest "k8s.io/autoscaler/cluster-autoscaler/processors/test" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" @@ -41,72 +42,80 @@ import ( ) func TestNodePoolAsyncInitialization(t *testing.T) { + scaleUpSize := 3 + failingNodeGroupName := "failing-ng" + provider := testprovider.NewTestCloudProvider( + func(nodeGroup string, increase int) error { + if nodeGroup == failingNodeGroupName { + return fmt.Errorf("Simulated error") + } + return nil + }, nil) + pod := BuildTestPod("p1", 2, 1000) + failingNodeGroup := provider.BuildNodeGroup(failingNodeGroupName, 0, 100, 0, false, true, "T1", nil) + successfulNodeGroup := provider.BuildNodeGroup("async-ng", 0, 100, 0, false, true, "T1", nil) + failedScaleUpErr := errors.ToAutoscalerError(errors.CloudProviderError, fmt.Errorf("Simulated error")).AddPrefix("failed to increase node group size: ") testCases := []struct { - name string - failingScaleUps map[string]bool - expectedScaleUps map[string]int + name string + nodeGroup *testprovider.TestNodeGroup + wantStatus status.ScaleUpStatus }{ { - name: "scale up upcoming node group", - expectedScaleUps: map[string]int{"async-ng": 3}, + name: "scale up upcoming node group", + nodeGroup: successfulNodeGroup, + wantStatus: status.ScaleUpStatus{ + Result: status.ScaleUpSuccessful, + ScaleUpInfos: []nodegroupset.ScaleUpInfo{ + { + Group: successfulNodeGroup, + CurrentSize: 0, + NewSize: scaleUpSize, + MaxSize: successfulNodeGroup.MaxSize(), + }, + }, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{ + {MainCreatedNodeGroup: successfulNodeGroup}, + }, + PodsTriggeredScaleUp: []*apiv1.Pod{pod}, + }, }, { - name: "failing initial scale up", - failingScaleUps: map[string]bool{"async-ng": true}, + name: "failing initial scale up", + nodeGroup: failingNodeGroup, + wantStatus: status.ScaleUpStatus{ + Result: status.ScaleUpError, + ScaleUpError: &failedScaleUpErr, + CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{ + {MainCreatedNodeGroup: failingNodeGroup}, + }, + FailedResizeNodeGroups: []cloudprovider.NodeGroup{failingNodeGroup}, + PodsTriggeredScaleUp: []*apiv1.Pod{pod}, + }, }, } + listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) + upcomingNodeGroup := provider.BuildNodeGroup("upcoming-ng", 0, 100, 0, false, true, "T1", nil) + options := config.AutoscalingOptions{AsyncNodeGroupsEnabled: true} + context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) + assert.NoError(t, err) + option := expander.Option{NodeGroup: upcomingNodeGroup, Pods: []*apiv1.Pod{pod}} + processors := processorstest.NewTestProcessors(&context) + processors.AsyncNodeGroupStateChecker = &asyncnodegroups.MockAsyncNodeGroupStateChecker{IsUpcomingNodeGroup: map[string]bool{upcomingNodeGroup.Id(): true}} + nodeInfo := framework.NewTestNodeInfo(BuildTestNode("t1", 100, 0)) + executor := newScaleUpExecutor(&context, processors.ScaleStateNotifier, processors.AsyncNodeGroupStateChecker) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - scaledUpGroups := make(map[string]int) - provider := testprovider.NewTestCloudProvider( - func(nodeGroup string, increase int) error { - if tc.failingScaleUps[nodeGroup] { - return fmt.Errorf("Simulated error") - } - scaledUpGroups[nodeGroup] += increase - return nil - }, nil) - options := config.AutoscalingOptions{ - AsyncNodeGroupsEnabled: true, - } - listers := kube_util.NewListerRegistry(nil, nil, nil, nil, nil, nil, nil, nil, nil) - context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) - assert.NoError(t, err) - p1 := BuildTestPod("p1", 2, 1000) - upcomingNodeGroup := provider.BuildNodeGroup("upcoming-ng", 0, 100, 0, false, true, "T1", nil) - createdNodeGroup := provider.BuildNodeGroup("async-ng", 0, 100, 0, false, true, "T1", nil) - option := expander.Option{ - NodeGroup: upcomingNodeGroup, - Pods: []*apiv1.Pod{p1}, - } - processors := processorstest.NewTestProcessors(&context) - processors.AsyncNodeGroupStateChecker = &asyncnodegroups.MockAsyncNodeGroupStateChecker{IsUpcomingNodeGroup: map[string]bool{upcomingNodeGroup.Id(): true}} - nodeInfo := framework.NewTestNodeInfo(BuildTestNode("t1", 100, 0)) - executor := newScaleUpExecutor(&context, processors.ScaleStateNotifier, processors.AsyncNodeGroupStateChecker) scaleUpStatusProcessor := &fakeScaleUpStatusProcessor{} initializer := NewAsyncNodeGroupInitializer(&option, nodeInfo, executor, taints.TaintConfig{}, nil, scaleUpStatusProcessor, &context, false) - initializer.SetTargetSize(upcomingNodeGroup.Id(), 3) + initializer.SetTargetSize(upcomingNodeGroup.Id(), int64(scaleUpSize)) asyncResult := nodegroups.AsyncNodeGroupCreationResult{ - CreationResult: nodegroups.CreateNodeGroupResult{MainCreatedNodeGroup: createdNodeGroup}, + CreationResult: nodegroups.CreateNodeGroupResult{MainCreatedNodeGroup: tc.nodeGroup}, CreatedToUpcomingMapping: map[string]string{ - createdNodeGroup.Id(): upcomingNodeGroup.Id(), + tc.nodeGroup.Id(): upcomingNodeGroup.Id(), }, } initializer.InitializeNodeGroup(asyncResult) - assert.Equal(t, len(scaledUpGroups), len(tc.expectedScaleUps)) - for groupName, increase := range tc.expectedScaleUps { - assert.Equal(t, increase, scaledUpGroups[groupName]) - } - if len(tc.failingScaleUps) > 0 { - expectedErr := errors.ToAutoscalerError(errors.CloudProviderError, fmt.Errorf("Simulated error")).AddPrefix("failed to increase node group size: ") - assert.Equal(t, scaleUpStatusProcessor.lastStatus, &status.ScaleUpStatus{ - Result: status.ScaleUpError, - ScaleUpError: &expectedErr, - CreateNodeGroupResults: []nodegroups.CreateNodeGroupResult{asyncResult.CreationResult}, - FailedResizeNodeGroups: []cloudprovider.NodeGroup{createdNodeGroup}, - PodsTriggeredScaleUp: option.Pods, - }) - } + assert.Equal(t, *scaleUpStatusProcessor.lastStatus, tc.wantStatus) }) } } diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index 68992602383..acf2c806ed7 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -627,6 +627,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr if scaleDownInCooldown { scaleDownStatus.Result = scaledownstatus.ScaleDownInCooldown + a.updateSoftDeletionTaints(allNodes) } else { klog.V(4).Infof("Starting scale down") @@ -645,21 +646,7 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr a.lastScaleDownDeleteTime = currentTime a.clusterStateRegistry.Recalculate() } - - if scaleDownStatus.Result == scaledownstatus.ScaleDownNoNodeDeleted && - a.AutoscalingContext.AutoscalingOptions.MaxBulkSoftTaintCount != 0 { - taintableNodes := a.scaleDownPlanner.UnneededNodes() - - // Make sure we are only cleaning taints from selected node groups. - selectedNodes := filterNodesFromSelectedGroups(a.CloudProvider, allNodes...) - - // This is a sanity check to make sure `taintableNodes` only includes - // nodes from selected nodes. - taintableNodes = intersectNodes(selectedNodes, taintableNodes) - untaintableNodes := subtractNodes(selectedNodes, taintableNodes) - actuation.UpdateSoftDeletionTaints(a.AutoscalingContext, taintableNodes, untaintableNodes) - } - + a.updateSoftDeletionTaints(allNodes) if typedErr != nil { klog.Errorf("Failed to scale down: %v", typedErr) a.lastScaleDownFailTime = currentTime @@ -679,6 +666,21 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) caerrors.AutoscalerErr return nil } +func (a *StaticAutoscaler) updateSoftDeletionTaints(allNodes []*apiv1.Node) { + if a.AutoscalingContext.AutoscalingOptions.MaxBulkSoftTaintCount != 0 { + taintableNodes := a.scaleDownPlanner.UnneededNodes() + + // Make sure we are only cleaning taints from selected node groups. + selectedNodes := filterNodesFromSelectedGroups(a.CloudProvider, allNodes...) + + // This is a sanity check to make sure `taintableNodes` only includes + // nodes from selected nodes. + taintableNodes = intersectNodes(selectedNodes, taintableNodes) + untaintableNodes := subtractNodes(selectedNodes, taintableNodes) + actuation.UpdateSoftDeletionTaints(a.AutoscalingContext, taintableNodes, untaintableNodes) + } +} + func (a *StaticAutoscaler) addUpcomingNodesToClusterSnapshot(upcomingCounts map[string]int, nodeInfosForGroups map[string]*framework.NodeInfo) error { nodeGroups := a.nodeGroupsById() upcomingNodeGroups := make(map[string]int) @@ -1001,7 +1003,7 @@ func filterNodesFromSelectedGroups(cp cloudprovider.CloudProvider, nodes ...*api filtered := make([]*apiv1.Node, 0, len(nodes)) for _, n := range nodes { if ng, err := cp.NodeGroupForNode(n); err != nil { - klog.Errorf("Failed to get a node group node node: %v", err) + klog.Errorf("Failed to get a nodegroup for node %q: %v", n.Name, err) } else if ng != nil { filtered = append(filtered, n) } diff --git a/cluster-autoscaler/core/static_autoscaler_test.go b/cluster-autoscaler/core/static_autoscaler_test.go index ec0e549dffd..d30536e0cdd 100644 --- a/cluster-autoscaler/core/static_autoscaler_test.go +++ b/cluster-autoscaler/core/static_autoscaler_test.go @@ -2674,6 +2674,65 @@ func TestStaticAutoscalerRunOnceInvokesScaleDownStatusProcessor(t *testing.T) { } +func TestFilterNodesFromSelectedGroups(t *testing.T) { + node1 := BuildTestNode("node1", 1000, 1000) + node1.Spec.ProviderID = "A" + node2 := BuildTestNode("node2", 1000, 1000) + node2.Spec.ProviderID = "B" + node3 := BuildTestNode("node3", 1000, 1000) + node3.Spec.ProviderID = "C" + invalidNode := BuildTestNode("invalidNode", 1000, 1000) + invalidNode.Spec.ProviderID = "invalid" + + provider := &mockprovider.CloudProvider{} + provider.On("NodeGroupForNode", mock.Anything).Return( + func(node *apiv1.Node) cloudprovider.NodeGroup { + if node.Spec.ProviderID == "A" || node.Spec.ProviderID == "B" { + return &mockprovider.NodeGroup{} + } + return nil + }, func(node *apiv1.Node) error { + if node.Spec.ProviderID == "invalid" { + return fmt.Errorf("broken provider") + } + return nil + }) + + tests := []struct { + name string + nodes []*apiv1.Node + wantNodes []*apiv1.Node + }{ + { + name: "returns no nodes if none were provided", + nodes: []*apiv1.Node{}, + wantNodes: []*apiv1.Node{}, + }, + { + name: "returns nodes with matching providers", + nodes: []*apiv1.Node{node1, node2}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + { + name: "filters out nodes with not matching provider", + nodes: []*apiv1.Node{node1, node2, node3}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + { + name: "filters out nodes with broken provider", + nodes: []*apiv1.Node{node1, node2, invalidNode}, + wantNodes: []*apiv1.Node{node1, node2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filteredNodes := filterNodesFromSelectedGroups(provider, tt.nodes...) + assert.Equal(t, tt.wantNodes, filteredNodes) + }) + } +} + func waitForDeleteToFinish(t *testing.T, deleteFinished <-chan bool) { t.Helper() select { diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index a3f48d6adc8..b3c406dea3e 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -16,7 +16,7 @@ require ( github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/skewer v0.0.14 + github.com/Azure/skewer v0.0.19 github.com/aws/aws-sdk-go v1.44.241 github.com/cenkalti/backoff/v4 v4.3.0 github.com/digitalocean/godo v1.27.0 diff --git a/cluster-autoscaler/go.sum b/cluster-autoscaler/go.sum index 6c8b4a5fb63..ebb2740482f 100644 --- a/cluster-autoscaler/go.sum +++ b/cluster-autoscaler/go.sum @@ -77,6 +77,8 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/skewer v0.0.14 h1:0mzUJhspECkajYyynYsOCp//E2PSnYXrgP45bcskqfQ= github.com/Azure/skewer v0.0.14/go.mod h1:6WTecuPyfGtuvS8Mh4JYWuHhO4kcWycGfsUBB+XTFG4= +github.com/Azure/skewer v0.0.19 h1:+qA1z8isKmlNkhAwZErNS2wD2jaemSk9NszYKr8dddU= +github.com/Azure/skewer v0.0.19/go.mod h1:LVH7jmduRKmPj8YcIz7V4f53xJEntjweL4aoLyChkwk= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/cluster-autoscaler/processors/podinjection/pod_group.go b/cluster-autoscaler/processors/podinjection/pod_group.go index 747e05a55f2..fa4c4bccfa6 100644 --- a/cluster-autoscaler/processors/podinjection/pod_group.go +++ b/cluster-autoscaler/processors/podinjection/pod_group.go @@ -57,7 +57,7 @@ func updatePodGroups(pod *apiv1.Pod, ownerRef metav1.OwnerReference, podGroups m if !found { return podGroups } - if group.sample == nil && pod.Spec.NodeName == "" { + if group.sample == nil || pod.CreationTimestamp.After(group.sample.CreationTimestamp.Time) { group.sample = pod group.ownerUid = ownerRef.UID } diff --git a/cluster-autoscaler/processors/podinjection/pod_injection_processor.go b/cluster-autoscaler/processors/podinjection/pod_injection_processor.go index 08645fa4d1d..82b44e0171f 100644 --- a/cluster-autoscaler/processors/podinjection/pod_injection_processor.go +++ b/cluster-autoscaler/processors/podinjection/pod_injection_processor.go @@ -93,6 +93,7 @@ func makeFakePods(ownerUid types.UID, samplePod *apiv1.Pod, podCount int) []*api newPod := withFakePodAnnotation(samplePod.DeepCopy()) newPod.Name = fmt.Sprintf("%s-copy-%d", samplePod.Name, i) newPod.UID = types.UID(fmt.Sprintf("%s-%d", string(ownerUid), i)) + newPod.Spec.NodeName = "" fakePods = append(fakePods, newPod) } return fakePods diff --git a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go index 4ce6b895b15..d709757255b 100644 --- a/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go +++ b/cluster-autoscaler/processors/podinjection/pod_injection_processor_test.go @@ -19,6 +19,7 @@ package podinjection import ( "fmt" "testing" + "time" "github.com/stretchr/testify/assert" @@ -40,24 +41,24 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { node := BuildTestNode("node1", 100, 0) replicaSet1 := createTestReplicaSet("rep-set-1", "default", 5) - scheduledPodRep1Copy1 := buildTestPod("default", "-scheduled-pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName(node.Name)) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + scheduledPodRep1Copy1 := buildTestPod("default", "-scheduled-pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName(node.Name)) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) job1 := createTestJob("job-1", "default", 10, 10, 0) - scheduledPodJob1Copy1 := buildTestPod("default", "scheduled-pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID), withNodeName(node.Name)) - podJob1Copy1 := buildTestPod("default", "pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID)) - podJob1Copy2 := buildTestPod("default", "pod-job1-2", withControllerOwnerRef(job1.Name, "Job", job1.UID)) + scheduledPodJob1Copy1 := buildTestPod("default", "scheduled-pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID), WithNodeName(node.Name)) + podJob1Copy1 := buildTestPod("default", "pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy2 := buildTestPod("default", "pod-job1-2", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) parallelStatefulset := createTestStatefulset("parallel-statefulset-1", "default", appsv1.ParallelPodManagement, 10) - scheduledParallelStatefulsetPod := buildTestPod("default", "parallel-scheduled-pod-statefulset-1", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID), withNodeName(node.Name)) - parallelStatefulsetPodCopy1 := buildTestPod("default", "parallel-pod-statefulset1-1", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) - parallelStatefulsetPodCopy2 := buildTestPod("default", "parallel-pod-statefulset1-2", withControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) + scheduledParallelStatefulsetPod := buildTestPod("default", "parallel-scheduled-pod-statefulset-1", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID), WithNodeName(node.Name)) + parallelStatefulsetPodCopy1 := buildTestPod("default", "parallel-pod-statefulset1-1", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) + parallelStatefulsetPodCopy2 := buildTestPod("default", "parallel-pod-statefulset1-2", WithControllerOwnerRef(parallelStatefulset.Name, "StatefulSet", parallelStatefulset.UID)) sequentialStatefulset := createTestStatefulset("sequential-statefulset-1", "default", appsv1.OrderedReadyPodManagement, 10) - scheduledSequentialStatefulsetPod := buildTestPod("default", "sequential-scheduled-pod-statefulset-1", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID), withNodeName(node.Name)) - sequentialStatefulsetPodCopy1 := buildTestPod("default", "sequential-pod-statefulset1-1", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) - sequentialStatefulsetPodCopy2 := buildTestPod("default", "sequential-pod-statefulset1-2", withControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) + scheduledSequentialStatefulsetPod := buildTestPod("default", "sequential-scheduled-pod-statefulset-1", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID), WithNodeName(node.Name)) + sequentialStatefulsetPodCopy1 := buildTestPod("default", "sequential-pod-statefulset1-1", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) + sequentialStatefulsetPodCopy2 := buildTestPod("default", "sequential-pod-statefulset1-2", WithControllerOwnerRef(sequentialStatefulset.Name, "StatefulSet", sequentialStatefulset.UID)) replicaSetLister, err := kubernetes.NewTestReplicaSetLister([]*appsv1.ReplicaSet{&replicaSet1}) assert.NoError(t, err) @@ -67,46 +68,46 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { assert.NoError(t, err) testCases := []struct { - name string - scheduledPods []*apiv1.Pod - unschedulabePods []*apiv1.Pod - wantPods []*apiv1.Pod + name string + scheduledPods []*apiv1.Pod + unschedulablePods []*apiv1.Pod + wantPods []*apiv1.Pod }{ { - name: "ReplicaSet", - scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1}, - unschedulabePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2}, - wantPods: append([]*apiv1.Pod{podRep1Copy1, podRep1Copy2}, makeFakePods(replicaSet1.UID, podRep1Copy1, 2)...), + name: "ReplicaSet", + scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1}, + unschedulablePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2}, + wantPods: append([]*apiv1.Pod{podRep1Copy1, podRep1Copy2}, makeFakePods(replicaSet1.UID, scheduledPodRep1Copy1, 2)...), }, { - name: "Job", - scheduledPods: []*apiv1.Pod{scheduledPodJob1Copy1}, - unschedulabePods: []*apiv1.Pod{podJob1Copy1, podJob1Copy2}, - wantPods: append([]*apiv1.Pod{podJob1Copy1, podJob1Copy2}, makeFakePods(job1.UID, podJob1Copy1, 7)...), + name: "Job", + scheduledPods: []*apiv1.Pod{scheduledPodJob1Copy1}, + unschedulablePods: []*apiv1.Pod{podJob1Copy1, podJob1Copy2}, + wantPods: append([]*apiv1.Pod{podJob1Copy1, podJob1Copy2}, makeFakePods(job1.UID, scheduledPodJob1Copy1, 7)...), }, { - name: "Statefulset - Parallel pod management policy", - scheduledPods: []*apiv1.Pod{scheduledParallelStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, - wantPods: append([]*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, makeFakePods(parallelStatefulset.UID, parallelStatefulsetPodCopy1, 7)...), + name: "Statefulset - Parallel pod management policy", + scheduledPods: []*apiv1.Pod{scheduledParallelStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, + wantPods: append([]*apiv1.Pod{parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, makeFakePods(parallelStatefulset.UID, scheduledParallelStatefulsetPod, 7)...), }, { - name: "Statefulset - sequential pod management policy", - scheduledPods: []*apiv1.Pod{scheduledSequentialStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, - wantPods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, + name: "Statefulset - sequential pod management policy", + scheduledPods: []*apiv1.Pod{scheduledSequentialStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, + wantPods: []*apiv1.Pod{sequentialStatefulsetPodCopy1, sequentialStatefulsetPodCopy2}, }, { - name: "Mix of controllers", - scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1, scheduledPodJob1Copy1, scheduledParallelStatefulsetPod}, - unschedulabePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, + name: "Mix of controllers", + scheduledPods: []*apiv1.Pod{scheduledPodRep1Copy1, scheduledPodJob1Copy1, scheduledParallelStatefulsetPod}, + unschedulablePods: []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, wantPods: append( append( append( []*apiv1.Pod{podRep1Copy1, podRep1Copy2, podJob1Copy1, podJob1Copy2, parallelStatefulsetPodCopy1, parallelStatefulsetPodCopy2}, - makeFakePods(replicaSet1.UID, podRep1Copy1, 2)...), - makeFakePods(job1.UID, podJob1Copy1, 7)...), - makeFakePods(parallelStatefulset.UID, parallelStatefulsetPodCopy1, 7)..., + makeFakePods(replicaSet1.UID, scheduledPodRep1Copy1, 2)...), + makeFakePods(job1.UID, scheduledPodJob1Copy1, 7)...), + makeFakePods(parallelStatefulset.UID, scheduledParallelStatefulsetPod, 7)..., ), }, } @@ -123,7 +124,7 @@ func TestTargetCountInjectionPodListProcessor(t *testing.T) { }, ClusterSnapshot: clusterSnapshot, } - pods, err := p.Process(&ctx, tc.unschedulabePods) + pods, err := p.Process(&ctx, tc.unschedulablePods) assert.NoError(t, err) assert.ElementsMatch(t, tc.wantPods, pods) }) @@ -134,31 +135,46 @@ func TestGroupPods(t *testing.T) { noControllerPod := buildTestPod("default", "pod-no-podGroup") replicaSet1 := createTestReplicaSet("rep-set-1", "default", 10) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1ScheduledCopy1 := buildTestPod("default", "pod-rep1-3", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName("n1")) - podRep1ScheduledCopy2 := buildTestPod("default", "pod-rep1-4", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), withNodeName("n1")) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1ScheduledCopy1 := buildTestPod("default", "pod-rep1-3", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName("n1")) + podRep1ScheduledCopy2 := buildTestPod("default", "pod-rep1-4", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), WithNodeName("n1")) + timestamp := time.Date(2025, 3, 19, 12, 0, 0, 0, time.UTC) + podRep1OlderCopy := buildTestPod( + "default", + "pod-rep1-5", + WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), + WithNodeName("n1"), + WithCreationTimestamp(timestamp), + ) + podRep1NewerCopy := buildTestPod( + "default", + "pod-rep1-6", + WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID), + WithNodeName("n1"), + WithCreationTimestamp(timestamp.Add(1*time.Minute)), + ) replicaSet2 := createTestReplicaSet("rep-set-2", "default", 10) - podRep2Copy1 := buildTestPod("default", "pod-rep2-1", withControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID)) - podRep2ScheduledCopy1 := buildTestPod("default", "pod-rep2-1", withControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID), withNodeName("n1")) + podRep2Copy1 := buildTestPod("default", "pod-rep2-1", WithControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID)) + podRep2ScheduledCopy1 := buildTestPod("default", "pod-rep2-1", WithControllerOwnerRef(replicaSet2.Name, "ReplicaSet", replicaSet2.UID), WithNodeName("n1")) replicaSet3 := createTestReplicaSet("rep-set-3", "default", 10) - podRep3Copy1 := buildTestPod("default", "pod-rep3-1", withControllerOwnerRef(replicaSet3.Name, "ReplicaSet", replicaSet3.UID)) + podRep3Copy1 := buildTestPod("default", "pod-rep3-1", WithControllerOwnerRef(replicaSet3.Name, "ReplicaSet", replicaSet3.UID)) job1 := createTestJob("job-1", "default", 10, 10, 0) - podJob1Copy1 := buildTestPod("default", "pod-job1-1", withControllerOwnerRef(job1.Name, "Job", job1.UID)) - podJob1Copy2 := buildTestPod("default", "pod-job1-2", withControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy1 := buildTestPod("default", "pod-job1-1", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) + podJob1Copy2 := buildTestPod("default", "pod-job1-2", WithControllerOwnerRef(job1.Name, "Job", job1.UID)) job2 := createTestJob("job-2", "default", 10, 10, 0) - podJob2Copy1 := buildTestPod("default", "pod-job-2", withControllerOwnerRef(job2.Name, "Job", job2.UID)) + podJob2Copy1 := buildTestPod("default", "pod-job-2", WithControllerOwnerRef(job2.Name, "Job", job2.UID)) statefulset1 := createTestStatefulset("statefulset-1", "default", appsv1.ParallelPodManagement, 10) - statefulset1Copy1 := buildTestPod("default", "pod-statefulset1-1", withControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) - statefulset1Copy2 := buildTestPod("default", "pod-statefulset1-2", withControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) + statefulset1Copy1 := buildTestPod("default", "pod-statefulset1-1", WithControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) + statefulset1Copy2 := buildTestPod("default", "pod-statefulset1-2", WithControllerOwnerRef(statefulset1.Name, "StatefulSet", statefulset1.UID)) statefulset2 := createTestStatefulset("statefulset-2", "default", appsv1.ParallelPodManagement, 10) - statefulset2Copy1 := buildTestPod("default", "pod-statefulset2-1", withControllerOwnerRef(statefulset2.Name, "StatefulSet", statefulset2.UID)) + statefulset2Copy1 := buildTestPod("default", "pod-statefulset2-1", WithControllerOwnerRef(statefulset2.Name, "StatefulSet", statefulset2.UID)) testCases := []struct { name string @@ -182,8 +198,8 @@ func TestGroupPods(t *testing.T) { scheduledPods: []*apiv1.Pod{podRep1ScheduledCopy1, podRep1ScheduledCopy2, podRep2ScheduledCopy1}, replicaSets: []*appsv1.ReplicaSet{&replicaSet1, &replicaSet2}, wantGroupedPods: map[types.UID]podGroup{ - replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: nil}, - replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: nil}, + replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1ScheduledCopy1, ownerUid: replicaSet1.UID}, + replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: podRep2ScheduledCopy1, ownerUid: replicaSet2.UID}, }, }, { @@ -192,7 +208,7 @@ func TestGroupPods(t *testing.T) { unscheduledPods: []*apiv1.Pod{podRep1Copy1, podRep2Copy1}, replicaSets: []*appsv1.ReplicaSet{&replicaSet1, &replicaSet2}, wantGroupedPods: map[types.UID]podGroup{ - replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1Copy1, ownerUid: replicaSet1.UID}, + replicaSet1.UID: {podCount: 2, desiredReplicas: 10, sample: podRep1ScheduledCopy2, ownerUid: replicaSet1.UID}, replicaSet2.UID: {podCount: 1, desiredReplicas: 10, sample: podRep2Copy1, ownerUid: replicaSet2.UID}, }, }, @@ -246,6 +262,15 @@ func TestGroupPods(t *testing.T) { statefulset1.UID: {podCount: 1, desiredReplicas: 10, sample: statefulset1Copy1, ownerUid: statefulset1.UID}, }, }, + { + name: "newest pod as a sample", + unscheduledPods: []*apiv1.Pod{podRep1Copy1}, + scheduledPods: []*apiv1.Pod{podRep1NewerCopy, podRep1OlderCopy}, + replicaSets: []*appsv1.ReplicaSet{&replicaSet1}, + wantGroupedPods: map[types.UID]podGroup{ + replicaSet1.UID: {podCount: 3, desiredReplicas: 10, sample: podRep1NewerCopy, ownerUid: replicaSet1.UID}, + }, + }, } for _, tc := range testCases { @@ -271,8 +296,8 @@ func TestGroupPods(t *testing.T) { func TestUpdatePodGroups(t *testing.T) { replicaSet1 := createTestReplicaSet("rep-set-1", "default", 10) - podRep1Copy1 := buildTestPod("default", "pod-rep1-1", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) - podRep1Copy2 := buildTestPod("default", "pod-rep1-2", withControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy1 := buildTestPod("default", "pod-rep1-1", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) + podRep1Copy2 := buildTestPod("default", "pod-rep1-2", WithControllerOwnerRef(replicaSet1.Name, "ReplicaSet", replicaSet1.UID)) samplePodGroups := map[types.UID]podGroup{replicaSet1.UID: makePodGroup(10)} sampleFalse := false sampleTrue := true @@ -347,7 +372,7 @@ func TestUpdatePodGroups(t *testing.T) { } } func TestMakeFakePods(t *testing.T) { - samplePod := buildTestPod("default", "test-pod") + samplePod := buildTestPod("default", "test-pod", WithNodeName("test-node")) // Test case: Positive fake pod count fakePodCount := 5 ownerUid := types.UID("sample uid") @@ -356,6 +381,7 @@ func TestMakeFakePods(t *testing.T) { for idx, fakePod := range fakePods { assert.Equal(t, fakePod.Name, fmt.Sprintf("%s-copy-%d", samplePod.Name, idx+1)) assert.Equal(t, fakePod.UID, types.UID(fmt.Sprintf("%s-%d", string(ownerUid), idx+1))) + assert.Equal(t, "", fakePod.Spec.NodeName) assert.NotNil(t, fakePod.Annotations) assert.Equal(t, fakePod.Annotations[FakePodAnnotationKey], FakePodAnnotationValue) } @@ -407,15 +433,3 @@ func buildTestPod(namespace, name string, opts ...podOption) *apiv1.Pod { } type podOption func(*apiv1.Pod) - -func withControllerOwnerRef(name, kind string, uid types.UID) podOption { - return func(pod *apiv1.Pod) { - pod.OwnerReferences = GenerateOwnerReferences(name, kind, "apps/v1", uid) - } -} - -func withNodeName(nodeName string) podOption { - return func(pod *apiv1.Pod) { - pod.Spec.NodeName = nodeName - } -} diff --git a/cluster-autoscaler/simulator/drain_test.go b/cluster-autoscaler/simulator/drain_test.go index bb7de730764..3b2faa84539 100644 --- a/cluster-autoscaler/simulator/drain_test.go +++ b/cluster-autoscaler/simulator/drain_test.go @@ -43,7 +43,11 @@ import ( func TestGetPodsToMove(t *testing.T) { var ( - testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + bspDisruptionTimeout = time.Hour + creationTimeBeforeBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(-time.Minute) + creationTimeAfterBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(time.Minute) + replicas = int32(5) unreplicatedPod = &apiv1.Pod{ @@ -68,6 +72,22 @@ func TestGetPodsToMove(t *testing.T) { OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), }, } + drainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeBeforeBspDisturptionTimeout}, + }, + } + nonDrainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeAfterBspDisturptionTimeout}, + }, + } localStoragePod = &apiv1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "localStoragePod", @@ -541,6 +561,28 @@ func TestGetPodsToMove(t *testing.T) { Reason: drain.UnmovableKubeSystemPod, }, }, + { + desc: "Kube-system no pdb system pods blocking", + pods: []*apiv1.Pod{nonDrainableBlockingSystemPod}, + wantErr: true, + wantBlocking: &drain.BlockingPod{ + Pod: nonDrainableBlockingSystemPod, + Reason: drain.UnmovableKubeSystemPod, + }}, + { + desc: "Kube-system no pdb system pods allowing", + pods: []*apiv1.Pod{drainableBlockingSystemPod}, + wantPods: []*apiv1.Pod{drainableBlockingSystemPod}, + }, + { + desc: "Kube-system no pdb system pods blocking", + pods: []*apiv1.Pod{drainableBlockingSystemPod, nonDrainableBlockingSystemPod}, + wantErr: true, + wantBlocking: &drain.BlockingPod{ + Pod: nonDrainableBlockingSystemPod, + Reason: drain.UnmovableKubeSystemPod, + }, + }, { desc: "Local storage", pods: []*apiv1.Pod{localStoragePod}, @@ -771,6 +813,7 @@ func TestGetPodsToMove(t *testing.T) { SkipNodesWithSystemPods: true, SkipNodesWithLocalStorage: true, SkipNodesWithCustomControllerPods: true, + BspDisruptionTimeout: bspDisruptionTimeout, } rules := append(tc.rules, rules.Default(deleteOptions)...) tracker := pdb.NewBasicRemainingPdbTracker() diff --git a/cluster-autoscaler/simulator/drainability/rules/rules.go b/cluster-autoscaler/simulator/drainability/rules/rules.go index be8d00cb415..e5103791299 100644 --- a/cluster-autoscaler/simulator/drainability/rules/rules.go +++ b/cluster-autoscaler/simulator/drainability/rules/rules.go @@ -65,7 +65,7 @@ func Default(deleteOptions options.NodeDeleteOptions) Rules { // Blocking checks {rule: replicated.New(deleteOptions.SkipNodesWithCustomControllerPods)}, - {rule: system.New(), skip: !deleteOptions.SkipNodesWithSystemPods}, + {rule: system.New(deleteOptions.BspDisruptionTimeout), skip: !deleteOptions.SkipNodesWithSystemPods}, {rule: notsafetoevict.New()}, {rule: localstorage.New(), skip: !deleteOptions.SkipNodesWithLocalStorage}, {rule: pdbrule.New()}, diff --git a/cluster-autoscaler/simulator/drainability/rules/system/rule.go b/cluster-autoscaler/simulator/drainability/rules/system/rule.go index fae89fb128a..adf2eca2fed 100644 --- a/cluster-autoscaler/simulator/drainability/rules/system/rule.go +++ b/cluster-autoscaler/simulator/drainability/rules/system/rule.go @@ -18,6 +18,7 @@ package system import ( "fmt" + "time" apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/simulator/drainability" @@ -25,12 +26,17 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/drain" ) +// KubeSystemNamespace is the namespase includes system pods +const KubeSystemNamespace = "kube-system" + // Rule is a drainability rule on how to handle system pods. -type Rule struct{} +type Rule struct { + BspDisruptionTimeout time.Duration +} // New creates a new Rule. -func New() *Rule { - return &Rule{} +func New(bspDisruptionTimeout time.Duration) *Rule { + return &Rule{BspDisruptionTimeout: bspDisruptionTimeout} } // Name returns the name of the rule. @@ -40,8 +46,20 @@ func (r *Rule) Name() string { // Drainable decides what to do with system pods on node drain. func (r *Rule) Drainable(drainCtx *drainability.DrainContext, pod *apiv1.Pod, _ *framework.NodeInfo) drainability.Status { - if pod.Namespace == "kube-system" && len(drainCtx.RemainingPdbTracker.MatchingPdbs(pod)) == 0 { + if isBlockingSystemPod(drainCtx, pod) { + if r.isBspPassedDisruptionTimeout(pod, drainCtx.Timestamp) { + return drainability.NewDrainableStatus() + } return drainability.NewBlockedStatus(drain.UnmovableKubeSystemPod, fmt.Errorf("non-daemonset, non-mirrored, non-pdb-assigned kube-system pod present: %s", pod.Name)) } return drainability.NewUndefinedStatus() } + +func isBlockingSystemPod(drainCtx *drainability.DrainContext, pod *apiv1.Pod) bool { + return pod.Namespace == KubeSystemNamespace && len(drainCtx.RemainingPdbTracker.MatchingPdbs(pod)) == 0 +} + +func (r *Rule) isBspPassedDisruptionTimeout(pod *apiv1.Pod, drainTime time.Time) bool { + return !pod.ObjectMeta.CreationTimestamp.IsZero() && + drainTime.After(pod.ObjectMeta.CreationTimestamp.Add(r.BspDisruptionTimeout)) +} diff --git a/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go b/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go index 9c34b808220..0dfcfb7888a 100644 --- a/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go +++ b/cluster-autoscaler/simulator/drainability/rules/system/rule_test.go @@ -34,7 +34,11 @@ import ( func TestDrainable(t *testing.T) { var ( - testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + testTime = time.Date(2020, time.December, 18, 17, 0, 0, 0, time.UTC) + bspDisruptionTimeout = time.Minute + creationTimeBeforeBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(-time.Second) + creationTimeAfterBspDisturptionTimeout = testTime.Add(-bspDisruptionTimeout).Add(time.Second) + replicas = int32(5) rc = apiv1.ReplicationController{ @@ -84,6 +88,24 @@ func TestDrainable(t *testing.T) { }, } + drainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: test.GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeBeforeBspDisturptionTimeout}, + }, + } + + nonDrainableBlockingSystemPod = &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "systemPod", + Namespace: "kube-system", + OwnerReferences: test.GenerateOwnerReferences("rs", "ReplicaSet", "extensions/v1beta1", ""), + CreationTimestamp: metav1.Time{Time: creationTimeAfterBspDisturptionTimeout}, + }, + } + emptyPDB = &policyv1.PodDisruptionBudget{} kubeSystemPDB = &policyv1.PodDisruptionBudget{ @@ -164,6 +186,18 @@ func TestDrainable(t *testing.T) { wantReason: drain.UnmovableKubeSystemPod, wantError: true, }, + "block non-pdb system pod existing for less than BspDisruptionTimeout": { + pod: nonDrainableBlockingSystemPod, + rcs: []*apiv1.ReplicationController{&kubeSystemRc}, + pdbs: []*policyv1.PodDisruptionBudget{emptyPDB}, + wantReason: drain.UnmovableKubeSystemPod, + wantError: true, + }, + "allow non-pdb system pod existing for more than BspDisruptionTimeout": { + pod: drainableBlockingSystemPod, + rcs: []*apiv1.ReplicationController{&kubeSystemRc}, + pdbs: []*policyv1.PodDisruptionBudget{kubeSystemPDB}, + }, } { t.Run(desc, func(t *testing.T) { tracker := pdb.NewBasicRemainingPdbTracker() @@ -173,7 +207,7 @@ func TestDrainable(t *testing.T) { RemainingPdbTracker: tracker, Timestamp: testTime, } - status := New().Drainable(drainCtx, test.pod, nil) + status := New(bspDisruptionTimeout).Drainable(drainCtx, test.pod, nil) assert.Equal(t, test.wantReason, status.BlockingReason) assert.Equal(t, test.wantError, status.Error != nil) }) diff --git a/cluster-autoscaler/simulator/options/nodedelete.go b/cluster-autoscaler/simulator/options/nodedelete.go index 6b6e17a1b7e..e56eb74f2f9 100644 --- a/cluster-autoscaler/simulator/options/nodedelete.go +++ b/cluster-autoscaler/simulator/options/nodedelete.go @@ -17,6 +17,8 @@ limitations under the License. package options import ( + "time" + "k8s.io/autoscaler/cluster-autoscaler/config" ) @@ -35,6 +37,9 @@ type NodeDeleteOptions struct { // set or replication controller should have to allow pod deletion during // scale down. MinReplicaCount int + // BspDisruptionTimeout is the timeout after which CA will evict + // non-pdb-assigned blocking system pods + BspDisruptionTimeout time.Duration } // NewNodeDeleteOptions returns new node delete options extracted from autoscaling options. @@ -44,5 +49,6 @@ func NewNodeDeleteOptions(opts config.AutoscalingOptions) NodeDeleteOptions { SkipNodesWithLocalStorage: opts.SkipNodesWithLocalStorage, SkipNodesWithCustomControllerPods: opts.SkipNodesWithCustomControllerPods, MinReplicaCount: opts.MinReplicaCount, + BspDisruptionTimeout: opts.BspDisruptionTimeout, } } diff --git a/cluster-autoscaler/utils/test/test_utils.go b/cluster-autoscaler/utils/test/test_utils.go index 577007744b8..0b9bfa7aafc 100644 --- a/cluster-autoscaler/utils/test/test_utils.go +++ b/cluster-autoscaler/utils/test/test_utils.go @@ -26,7 +26,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/stretchr/testify/mock" - apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -113,6 +112,13 @@ func WithResourceClaim(refName, claimName, templateName string) func(*apiv1.Pod) } } +// WithControllerOwnerRef sets an owner reference to the pod. +func WithControllerOwnerRef(name, kind string, uid types.UID) func(*apiv1.Pod) { + return func(pod *apiv1.Pod) { + pod.OwnerReferences = GenerateOwnerReferences(name, kind, "apps/v1", uid) + } +} + // WithDSController creates a daemonSet owner ref for the pod. func WithDSController() func(*apiv1.Pod) { return func(pod *apiv1.Pod) { @@ -174,6 +180,13 @@ func WithMaxSkew(maxSkew int32, topologySpreadingKey string) func(*apiv1.Pod) { } } +// WithCreationTimestamp sets creation timestamp to the pod. +func WithCreationTimestamp(timestamp time.Time) func(*apiv1.Pod) { + return func(pod *apiv1.Pod) { + pod.CreationTimestamp = metav1.Time{Time: timestamp} + } +} + // WithDeletionTimestamp sets deletion timestamp to the pod. func WithDeletionTimestamp(deletionTimestamp time.Time) func(*apiv1.Pod) { return func(pod *apiv1.Pod) { diff --git a/vertical-pod-autoscaler/OWNERS b/vertical-pod-autoscaler/OWNERS index 72107eb3bd5..bc51f4f1e34 100644 --- a/vertical-pod-autoscaler/OWNERS +++ b/vertical-pod-autoscaler/OWNERS @@ -3,6 +3,7 @@ approvers: - jbartosik - voelzmo - raywainman +- adrianmoisey reviewers: - kwiesmueller - jbartosik diff --git a/vertical-pod-autoscaler/RELEASE.md b/vertical-pod-autoscaler/RELEASE.md index 9b7fc80b981..19d5b1c0ae0 100644 --- a/vertical-pod-autoscaler/RELEASE.md +++ b/vertical-pod-autoscaler/RELEASE.md @@ -9,7 +9,7 @@ Before doing the release for the first time check if you have all the necessary There are the following steps of the release process: 1. [ ] Open issue to track the release. -2. [ ] Update VPA version const. +2. [ ] Rollup all changes. 3. [ ] Build and stage images. 4. [ ] Test the release. 5. [ ] Promote image. @@ -20,7 +20,7 @@ There are the following steps of the release process: Open a new issue to track the release, use the [vpa_release](https://github.com/kubernetes/autoscaler/issues/new?&template=vpa_release.md) template. We use the issue to communicate what is state of the release. -## Update VPA version const +## Rollup all changes 1. [ ] Wait for all VPA changes that will be in the release to merge. 2. [ ] Wait for [the end to end tests](https://testgrid.k8s.io/sig-autoscaling-vpa) to run with all VPA changes @@ -31,13 +31,12 @@ We use the issue to communicate what is state of the release. ### New minor release -1. [ ] Change the version in - [common/version-go](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/common/version.go) - to `1.${next-minor}.0`, -2. [ ] Commit and merge the change, -3. [ ] Go to the merged change, -4. [ ] [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) named `vpa-release-1.${next-minor}` from the +1. [ ] [Create a new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository) named `vpa-release-1.${next-minor}` from the merged change. +2. [ ] In the **main branch**, change the version in + [common/version-go](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/common/version.go) + to `1.${next-minor}.0`. +3. [ ] Commit and merge the change. ### New patch release diff --git a/vertical-pod-autoscaler/common/version.go b/vertical-pod-autoscaler/common/version.go index cfd4803cd8e..06a88725a66 100644 --- a/vertical-pod-autoscaler/common/version.go +++ b/vertical-pod-autoscaler/common/version.go @@ -21,7 +21,7 @@ package common var gitCommit = "" // versionCore is the version of VPA. -const versionCore = "1.3.0" +const versionCore = "1.4.0" // VerticalPodAutoscalerVersion returns the version of the VPA. func VerticalPodAutoscalerVersion() string { diff --git a/vertical-pod-autoscaler/docs/features.md b/vertical-pod-autoscaler/docs/features.md index bc1f8d90f48..1a15863936b 100644 --- a/vertical-pod-autoscaler/docs/features.md +++ b/vertical-pod-autoscaler/docs/features.md @@ -4,6 +4,8 @@ - [Limits control](#limits-control) - [Memory Value Humanization](#memory-value-humanization) +- [CPU Recommendation Rounding](#cpu-recommendation-rounding) +- [In-Place Updates](#in-place-updates-inplaceorrecreate) ## Limits control @@ -50,4 +52,78 @@ To enable this feature, set the --round-cpu-millicores flag when running the VPA ```bash --round-cpu-millicores=50 -``` \ No newline at end of file +``` + +## In-Place Updates (`InPlaceOrRecreate`) + +> [!WARNING] +> FEATURE STATE: VPA v1.4.0 [alpha] + +VPA supports in-place updates to reduce disruption when applying resource recommendations. This feature leverages Kubernetes' in-place update capabilities (beta in 1.33) to modify container resources without requiring pod recreation. +For more information, see [AEP-4916: Support for in place updates in VPA](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support) + +### Usage + +To use in-place updates, set the VPA's `updateMode` to `InPlaceOrRecreate`: +```yaml +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: my-vpa +spec: + updatePolicy: + updateMode: "InPlaceOrRecreate" +``` + +### Behavior + +When using `InPlaceOrRecreate` mode, VPA will first attempt to apply updates in-place, if in-place update fails, VPA will fall back to pod recreation. +Updates are attempted when: +* Container requests are outside the recommended bounds +* Quick OOM occurs +* For long-running pods (>12h), when recommendations differ significantly (>10%) + +Important Notes + +* Disruption Possibility: While in-place updates aim to minimize disruption, they cannot guarantee zero disruption as the container runtime is responsible for the actual resize operation. + +* Memory Limit Downscaling: In the beta version, memory limit downscaling is not supported for pods with resizePolicy: PreferNoRestart. In such cases, VPA will fall back to pod recreation. + +### Requirements: + +* Kubernetes 1.33+ with `InPlacePodVerticalScaling` feature gate enabled +* VPA version 1.4.0+ with `InPlaceOrRecreate` feature gate enabled + +### Configuration + +Enable the feature by setting the following flags in VPA components ( for both updater and admission-controller ): + +```bash +--feature-gates=InPlaceOrRecreate=true +``` + +### Limitations + +* All containers in a pod are updated together (partial updates not supported) +* Memory downscaling requires careful consideration to prevent OOMs +* Updates still respect VPA's standard update conditions and timing restrictions +* In-place updates will fail if they would result in a change to the pod's QoS class + +### Fallback Behavior + +VPA will fall back to pod recreation in the following scenarios: + +* In-place update is [infeasible](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#resize-status) (node resources, etc.) +* Update is [deferred](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#resize-status) for more than 5 minutes +* Update is in progress for more than 1 hour +* [Pod QoS](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/) class would change due to the update +* Memory limit downscaling is required with [PreferNoRestart policy](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md#container-resize-policy) + +### Monitoring + +VPA provides metrics to track in-place update operations: + +* `vpa_in_place_updatable_pods_total`: Number of pods matching in-place update criteria +* `vpa_in_place_updated_pods_total`: Number of pods successfully updated in-place +* `vpa_vpas_with_in_place_updatable_pods_total`: Number of VPAs with pods eligible for in-place updates +* `vpa_vpas_with_in_place_updated_pods_total`: Number of VPAs with successfully in-place updated pods \ No newline at end of file diff --git a/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md b/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md index f3d56424581..70bf7cd8aef 100644 --- a/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md +++ b/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md @@ -170,7 +170,8 @@ VPA updater will consider that the update failed if: * The pod has condition `PodResizePending` with reason `Infeasible` or * The pod has condition `PodResizePending` with reason `Deferred` and more than 5 minutes elapsed since the update or -* The pod has condition `PodResizing` and more than 1 hour elapsed since the update or +* The pod has condition `PodResizeInProgress` and more than 1 hour elapsed since + the update or * Patch attempt returns an error. Note that in the initial version of In-Place updates, memory limit downscaling will always fail @@ -228,7 +229,7 @@ Today, VPA updater considers the following conditions when deciding if it should * Outside recommended range, * Long-lived pod with significant change. * `EvictionRequirements` are all true. - + `InPlaceOrRecreate` will attempt to apply an update in place if it meets at least one of the following conditions: * Quick OOM, @@ -251,7 +252,7 @@ of the following conditions: The following test scenarios will be added to e2e tests. The `InPlaceOrRecreate` mode will be tested in the following scenarios: -* Admission controller applies recommendation to pod controlled by VPA. +* Admission controller applies recommendation to pod controlled by VPA. * In-place update applied to all containers of a pod. * In-place update will fail. Pod should be evicted and the recommendation applied. * In-place update will fail but `CanEvict` is false, pod should not be evicted. diff --git a/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md new file mode 100644 index 00000000000..8bc236253a8 --- /dev/null +++ b/vertical-pod-autoscaler/enhancements/7862-cpu-startup-boost/README.md @@ -0,0 +1,312 @@ +# AEP-7862: CPU Startup Boost + + +- [AEP-7862: CPU Startup Boost](#aep-7862-cpu-startup-boost) + - [Summary](#summary) + - [Goals](#goals) + - [Non-Goals](#non-goals) + - [Proposal](#proposal) + - [Design Details](#design-details) + - [Workflow](#workflow) + - [API Changes](#api-changes) + - [Priority of `StartupBoost`](#priority-of-startupboost) + - [Validation](#validation) + - [Static Validation](#static-validation) + - [Dynamic Validation](#dynamic-validation) + - [Mitigating Failed In-Place Downsizes](#mitigating-failed-in-place-downsizes) + - [Feature Enablement and Rollback](#feature-enablement-and-rollback) + - [How can this feature be enabled / disabled in a live cluster?](#how-can-this-feature-be-enabled--disabled-in-a-live-cluster) + - [Kubernetes Version Compatibility](#kubernetes-version-compatibility) + - [Test Plan](#test-plan) + - [Examples](#examples) + - [CPU Boost Only](#cpu-boost-only) + - [CPU Boost and Vanilla VPA](#cpu-boost-and-vanilla-vpa) + - [Implementation History](#implementation-history) + + +## Summary + +Long application start time is a known problem for more traditional workloads +running in containerized applications, especially Java workloads. This delay can +negatively impact the user experience and overall application performance. One +potential solution is to provide additional CPU resources to pods during their +startup phase, but this can lead to waste if the extra CPU resources are not +set back to their original values after the pods have started up. + +This proposal allows VPA to boost the CPU request and limit of containers during +the pod startup and to scale the CPU resources back down when the pod is +`Ready` or after certain time has elapsed, leveraging the +[in-place pod resize Kubernetes feature](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). + +> [!NOTE] +> This feature depends on the new `InPlaceOrRecreate` VPA mode: +> [AEP-4016: Support for in place updates in VPA](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md) + +### Goals + +* Allow VPA to boost the CPU request and limit of a pod's containers during the +pod (re-)creation time. +* Allow VPA to scale pods down [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources) +to the existing VPA recommendation for that container, if any, or to the CPU +resources configured in the pod spec, as soon as their [`Ready`](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions) +condition is true and `StartupBoost.CPU.Duration` has elapsed. + +### Non-Goals + +* Allow VPA to boost CPU resources of pods outside of the pod (re-)creation +time. +* Allow VPA to boost memory resources. + * This is out of scope for now because the in-place pod resize feature + [does not support memory limit decrease yet.](https://github.com/kubernetes/enhancements/tree/758ea034908515a934af09d03a927b24186af04c/keps/sig-node/1287-in-place-update-pod-resources#memory-limit-decreases) + +## Proposal + +* To extend [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191) +with a new `StartupBoost` field to allow users to configure the CPU startup +boost. + +* To extend [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) +with a new `StartupBoostOnly` mode to allow users to only enable the startup +boost feature and not vanilla VPA altogether. + +* To allow CPU startup boost if a `StartupBoost` config is specified in `Auto` +[`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236) +container policies. + +## Design Details + +### Workflow + +1. The user first configures the CPU startup boost on their VPA object + +1. When a pod targeted by that VPA is created, the kube-apiserver invokes the +VPA Admission Controller + +1. The VPA Admission Controller modifies the pod's containers CPU request and +limits to align with its `StartupBoost` policy, if specified, during the pod +creation. + +1. The VPA Updater monitors pods targeted by the VPA object and when the pod +condition is `Ready` and `StartupBoost.CPU.Duration` has elapsed, it scales +down the CPU resources to the appropriate non-boosted value: +`existing VPA recommendation for that container` (if any) OR the +`CPU resources configured in the pod spec`. + * The scale down is applied [in-place](https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/1287-in-place-update-pod-resources). + +### API Changes + +The new `StartupBoost` parameter will be added to the [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191) +and contain the following fields: + * `StartupBoost.CPU.Factor`: the factor by which to multiply the initial + resource request and limit of the containers' targeted by the VPA object. + * `StartupBoost.CPU.Value`: the target value of the CPU request or limit + during the startup boost phase. + * [Optional] `StartupBoost.CPU.Duration`: if specified, it indicates for how + long to keep the pod boosted **after** it goes to `Ready`. + +> [!IMPORTANT] +> The boosted CPU value will be capped by +> [`--container-recommendation-max-allowed-cpu`](https://github.com/kubernetes/autoscaler/blob/4d294562e505431d518a81e8833accc0ec99c9b8/vertical-pod-autoscaler/pkg/recommender/main.go#L122) +> flag value, if set. + +> [!IMPORTANT] +> Only one of `Factor` or `Value` may be specified per container policy. + + +> [!NOTE] +> To ensure that containers are unboosted only after their applications are +> started and ready, it is recommended to configure a +> [Readiness or a Startup probe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +> for the containers that will be CPU boosted. Check the [Test Plan](#test-plan) +> section for more details on this feature's behavior for different combinations +> of probers + `StartupBoost.CPU.Duration`. + +We will also add a new mode to the [`ContainerScalingMode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L231-L236): + * **NEW**: `StartupBoostOnly`: new mode that will allow users to only enable + the startup boost feature for a container and not vanilla VPA altogether. + * **NEW**: `Auto`: we will modify the existing `Auto` mode to enable both + vanilla VPA and CPU Startup Boost (when `StartupBoost` parameter is + specified). + +#### Priority of `StartupBoost` + +The new `StartupBoost` field will take precedence over the rest of the container +resource policy configurations. Functioning independently from all other fields +in [`ContainerResourcePolicy`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L191), +**except for**: + * [`ContainerName`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L192-L195) + * [`Mode`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L196-L198) + * [`ControlledValues`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L214-L217) + +This means that a container's CPU request/limit can be boosted during startup +beyond [`MaxAllowed`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L203-L206), +for example, or it will be able to be boosted even if CPU is explicitly +excluded from [`ControlledResources`](https://github.com/kubernetes/autoscaler/blob/vertical-pod-autoscaler-1.3.0/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1/types.go#L208-L212). + +### Validation + +#### Static Validation + +* We will check that the `startupBoost` configuration is valid when VPA objects +are created/updated: + * The VPA autoscaling mode must be `InPlaceOrRecreate` (since it does not + make sense to use this feature with disruptive modes of VPA). + * The boost factor is >= 1 (via CRD validation rules) + * Only one of `StartupBoost.CPU.Factor` or `StartupBoost.CPU.Value` is + specified + * The [feature enablement](#feature-enablement) flags must be on. + + +#### Dynamic Validation + +* `StartupBoost.CPU.Value` must be greater than the CPU request or limit of the + container during the boost phase, otherwise we risk downscaling the container. + +### Mitigating Failed In-Place Downsizes + +The VPA Updater **will not** evict a pod if it attempted to scaled the pod down +in place (to unboost its CPU resources) and the update failed (see the +[scenarios](https://github.com/kubernetes/autoscaler/blob/0a34bf5d3a71b486bdaa440f1af7f8d50dc8e391/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md?plain=1#L164-L169 ) where the VPA +updater will consider that the update failed). This is to avoid an eviction +loop: + +1. A pod is created and has its CPU resources boosted +1. The pod meets the conditions to be unboosted. VPA Updater tries to downscale +the pod in-place and it fails. +1. VPA Updater evicts the pod. Logic flow goes back to (1). + +### Feature Enablement and Rollback + +#### How can this feature be enabled / disabled in a live cluster? + +* Feature gates names: `CPUStartupBoost` and `InPlaceOrRecreate` (from +[AEP-4016](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support/README.md#feature-enablement-and-rollback)) +* Components depending on the feature gates: + * admission-controller + * updater + +Enabling of feature gates `CPUStartupBoost` AND `InPlaceOrRecreate` will cause +the following to happen: + * admission-controller to **accept** new VPA objects being created with +`StartupBoostOnly` configured. + * admission-controller to **boost** CPU resources. + * updater to **unboost** the CPU resources. + +Disabling of feature gates `CPUStartupBoost` OR `InPlaceOrRecreate` will cause +the following to happen: + * admission-controller to **reject** new VPA objects being created with + `StartupBoostOnly` configured. + * A descriptive error message should be returned to the user letting them + know that they are using a feature gated feature. + * admission-controller **to not** boost CPU resources, should it encounter a + VPA configured with a `StartupBoost` config and `StartupBoostOnly` or `Auto` + `ContainerScalingMode`. + * updater **to not** unboost CPU resources when pods meet the scale down + requirements, should it encounter a VPA configured with a `StartupBoost` + config and `StartupBoostOnly` or `Auto` `ContainerScalingMode`. + +### Kubernetes Version Compatibility + +Similarly to [AEP-4016](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler/enhancements/4016-in-place-updates-support#kubernetes-version-compatibility), +`StartupBoost` configuration and `StartupBoostOnly` mode are built assuming that +VPA will be running on a Kubernetes 1.33+ with the beta version of +[KEP-1287: In-Place Update of Pod Resources](https://github.com/kubernetes/enhancements/issues/1287) +enabled. If this is not the case, VPA's attempt to unboost pods may fail and the +pods may remain boosted for their whole lifecycle. + +## Test Plan + +Other than comprehensive unit tests, we will also add the following scenarios to +our e2e tests: + +* CPU Startup Boost recommendation is applied to pod controlled by VPA until it +becomes `Ready` and `StartupBoost.CPU.Duration` has elapsed. Then, the pod is +scaled back down in-place. We'll also test the following sub-cases: + * Boost is applied to all containers of a pod. + * Boost is applied only to a subset of containers in a pod. + * Combinations of probes + `StartupBoost.CPU.Duration`: + * No probes and no `StartupBoost.CPU.Duration` specified: unboost will + likely happen immediately. + * No probes and a 60s `StartupBoost.CPU.Duration`: unboost will likely + happen after 60s. + * A readiness/startup probe and no `StartupBoost.CPU.Duration` specified: + unboost will likely as soon as the pod becomes `Ready`. + * A readiness/startup probe and a 60s `StartupBoost.CPU.Duration` + specified: unboost will likely happen 60s **after** the pod becomes `Ready`. + +* Pod is not evicted if the in-place update fails when scaling the pod back +down. + +## Examples + +Here are some examples of the VPA CR incorporating CPU boosting for different +scenarios. + +### CPU Boost Only + +All containers under `example` deployment will receive "regular" VPA updates, +**except for** `boosted-container-name`. `boosted-container-name` will only be +CPU boosted/unboosted, because it has a `StartupBoostOnly` container policy. + +```yaml +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: example-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: example + updatePolicy: + # VPA Update mode must be InPlaceOrRecreate + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + - containerName: "boosted-container-name" + mode: "StartupBoostOnly" + startupBoost: + cpu: + factor: 2.0 +``` + +### CPU Boost and Vanilla VPA + +All containers under `example` deployment will receive "regular" VPA updates, +**including** `boosted-container-name`. Additionally, `boosted-container-name` +will be CPU boosted/unboosted, because it has a `StartupBoost` config in its +container policy and `Auto` container policy mode. + +```yaml +apiVersion: "autoscaling.k8s.io/v1" +kind: VerticalPodAutoscaler +metadata: + name: example-vpa +spec: + targetRef: + apiVersion: "apps/v1" + kind: Deployment + name: example + updatePolicy: + # VPA Update mode must be InPlaceOrRecreate + updateMode: "InPlaceOrRecreate" + resourcePolicy: + containerPolicies: + - containerName: "boosted-container-name" + mode: "Auto" # Vanilla VPA mode + Startup Boost + minAllowed: + cpu: "250m" + memory: "100Mi" + maxAllowed: + cpu: "500m" + memory: "600Mi" + # The CPU boosted resources can go beyond maxAllowed. + startupBoost: + cpu: + value: 4 +``` + +## Implementation History + +* 2025-03-20: Initial version. + diff --git a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go index db3b5d70ff6..ca97df21d62 100644 --- a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go +++ b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer.go @@ -42,11 +42,11 @@ type CheckpointWriter interface { type checkpointWriter struct { vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter - cluster *model.ClusterState + cluster model.ClusterState } // NewCheckpointWriter returns new instance of a CheckpointWriter -func NewCheckpointWriter(cluster *model.ClusterState, vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter) CheckpointWriter { +func NewCheckpointWriter(cluster model.ClusterState, vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter) CheckpointWriter { return &checkpointWriter{ vpaCheckpointClient: vpaCheckpointClient, cluster: cluster, @@ -77,7 +77,7 @@ func getVpasToCheckpoint(clusterVpas map[model.VpaID]*model.Vpa) []*model.Vpa { } func (writer *checkpointWriter) StoreCheckpoints(ctx context.Context, now time.Time, minCheckpoints int) error { - vpas := getVpasToCheckpoint(writer.cluster.Vpas) + vpas := getVpasToCheckpoint(writer.cluster.VPAs()) for _, vpa := range vpas { // Draining ctx.Done() channel. ctx.Err() will be checked if timeout occurred, but minCheckpoints have @@ -123,13 +123,13 @@ func (writer *checkpointWriter) StoreCheckpoints(ctx context.Context, now time.T // Build the AggregateContainerState for the purpose of the checkpoint. This is an aggregation of state of all // containers that belong to pods matched by the VPA. // Note however that we exclude the most recent memory peak for each container (see below). -func buildAggregateContainerStateMap(vpa *model.Vpa, cluster *model.ClusterState, now time.Time) map[string]*model.AggregateContainerState { +func buildAggregateContainerStateMap(vpa *model.Vpa, cluster model.ClusterState, now time.Time) map[string]*model.AggregateContainerState { aggregateContainerStateMap := vpa.AggregateStateByContainerName() // Note: the memory peak from the current (ongoing) aggregation interval is not included in the // checkpoint to avoid having multiple peaks in the same interval after the state is restored from // the checkpoint. Therefore we are extracting the current peak from all containers. // TODO: Avoid the nested loop over all containers for each VPA. - for _, pod := range cluster.Pods { + for _, pod := range cluster.Pods() { for containerName, container := range pod.Containers { aggregateKey := cluster.MakeAggregateStateKey(pod, containerName) if vpa.UsesAggregation(aggregateKey) { diff --git a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go index 6828be9ad46..c9dc14fc79f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/checkpoint/checkpoint_writer_test.go @@ -54,7 +54,7 @@ var ( const testGcPeriod = time.Minute -func addVpa(t *testing.T, cluster *model.ClusterState, vpaID model.VpaID, selector string) *model.Vpa { +func addVpa(t *testing.T, cluster model.ClusterState, vpaID model.VpaID, selector string) *model.Vpa { var apiObject vpa_types.VerticalPodAutoscaler apiObject.Namespace = vpaID.Namespace apiObject.Name = vpaID.VpaName @@ -64,7 +64,7 @@ func addVpa(t *testing.T, cluster *model.ClusterState, vpaID model.VpaID, select if err != nil { t.Fatalf("AddOrUpdateVpa() failed: %v", err) } - return cluster.Vpas[vpaID] + return cluster.VPAs()[vpaID] } func TestMergeContainerStateForCheckpointDropsRecentMemoryPeak(t *testing.T) { diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go index 587581dd194..8d7855b385f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder.go @@ -55,7 +55,7 @@ const ( DefaultRecommenderName = "default" ) -// ClusterStateFeeder can update state of ClusterState object. +// ClusterStateFeeder can update state of clusterState object. type ClusterStateFeeder interface { // InitFromHistoryProvider loads historical pod spec into clusterState. InitFromHistoryProvider(historyProvider history.HistoryProvider) @@ -78,7 +78,7 @@ type ClusterStateFeeder interface { // ClusterStateFeederFactory makes instances of ClusterStateFeeder. type ClusterStateFeederFactory struct { - ClusterState *model.ClusterState + ClusterState model.ClusterState KubeClient kube_client.Interface MetricsClient metrics.MetricsClient VpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter @@ -202,7 +202,7 @@ type clusterStateFeeder struct { oomChan <-chan oom.OomInfo vpaCheckpointClient vpa_api.VerticalPodAutoscalerCheckpointsGetter vpaLister vpa_lister.VerticalPodAutoscalerLister - clusterState *model.ClusterState + clusterState model.ClusterState selectorFetcher target.VpaTargetSelectorFetcher memorySaveMode bool controllerFetcher controllerfetcher.ControllerFetcher @@ -244,7 +244,7 @@ func (feeder *clusterStateFeeder) InitFromHistoryProvider(historyProvider histor func (feeder *clusterStateFeeder) setVpaCheckpoint(checkpoint *vpa_types.VerticalPodAutoscalerCheckpoint) error { vpaID := model.VpaID{Namespace: checkpoint.Namespace, VpaName: checkpoint.Spec.VPAObjectName} - vpa, exists := feeder.clusterState.Vpas[vpaID] + vpa, exists := feeder.clusterState.VPAs()[vpaID] if !exists { return fmt.Errorf("cannot load checkpoint to missing VPA object %s/%s", vpaID.Namespace, vpaID.VpaName) } @@ -263,7 +263,7 @@ func (feeder *clusterStateFeeder) InitFromCheckpoints() { feeder.LoadVPAs(context.TODO()) namespaces := make(map[string]bool) - for _, v := range feeder.clusterState.Vpas { + for _, v := range feeder.clusterState.VPAs() { namespaces[v.ID.Namespace] = true } @@ -431,15 +431,15 @@ func (feeder *clusterStateFeeder) LoadVPAs(ctx context.Context) { for _, condition := range conditions { if condition.delete { - delete(feeder.clusterState.Vpas[vpaID].Conditions, condition.conditionType) + delete(feeder.clusterState.VPAs()[vpaID].Conditions, condition.conditionType) } else { - feeder.clusterState.Vpas[vpaID].Conditions.Set(condition.conditionType, true, "", condition.message) + feeder.clusterState.VPAs()[vpaID].Conditions.Set(condition.conditionType, true, "", condition.message) } } } } // Delete non-existent VPAs from the model. - for vpaID := range feeder.clusterState.Vpas { + for vpaID := range feeder.clusterState.VPAs() { if _, exists := vpaKeys[vpaID]; !exists { klog.V(3).InfoS("Deleting VPA", "vpa", klog.KRef(vpaID.Namespace, vpaID.VpaName)) if err := feeder.clusterState.DeleteVpa(vpaID); err != nil { @@ -447,7 +447,7 @@ func (feeder *clusterStateFeeder) LoadVPAs(ctx context.Context) { } } } - feeder.clusterState.ObservedVpas = vpaCRDs + feeder.clusterState.SetObservedVPAs(vpaCRDs) } // LoadPods loads pod into the cluster state. @@ -460,7 +460,7 @@ func (feeder *clusterStateFeeder) LoadPods() { for _, spec := range podSpecs { pods[spec.ID] = spec } - for key := range feeder.clusterState.Pods { + for key := range feeder.clusterState.Pods() { if _, exists := pods[key]; !exists { klog.V(3).InfoS("Deleting Pod", "pod", klog.KRef(key.Namespace, key.PodName)) feeder.clusterState.DeletePod(key) @@ -476,6 +476,11 @@ func (feeder *clusterStateFeeder) LoadPods() { klog.V(0).InfoS("Failed to add container", "container", container.ID, "error", err) } } + for _, initContainer := range pod.InitContainers { + podInitContainers := feeder.clusterState.Pods()[pod.ID].InitContainers + feeder.clusterState.Pods()[pod.ID].InitContainers = append(podInitContainers, initContainer.ID.ContainerName) + + } } } @@ -488,6 +493,12 @@ func (feeder *clusterStateFeeder) LoadRealTimeMetrics() { sampleCount := 0 droppedSampleCount := 0 for _, containerMetrics := range containersMetrics { + podInitContainers := feeder.clusterState.Pods()[containerMetrics.ID.PodID].InitContainers + if slices.Contains(podInitContainers, containerMetrics.ID.ContainerName) { + klog.V(3).InfoS("Skipping metric samples for init container", "pod", klog.KRef(containerMetrics.ID.PodID.Namespace, containerMetrics.ID.PodID.PodName), "container", containerMetrics.ID.ContainerName) + droppedSampleCount += len(containerMetrics.Usage) + continue + } for _, sample := range newContainerUsageSamplesWithKey(containerMetrics) { if err := feeder.clusterState.AddSample(sample); err != nil { // Not all pod states are tracked in memory saver mode @@ -518,7 +529,7 @@ Loop: } func (feeder *clusterStateFeeder) matchesVPA(pod *spec.BasicPodSpec) bool { - for vpaKey, vpa := range feeder.clusterState.Vpas { + for vpaKey, vpa := range feeder.clusterState.VPAs() { podLabels := labels.Set(pod.PodLabels) if vpaKey.Namespace == pod.ID.Namespace && vpa.PodSelector.Matches(podLabels) { return true diff --git a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go index af50c4388b5..7705cb64212 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/cluster_feeder_test.go @@ -36,6 +36,7 @@ import ( vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" fakeautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned/typed/autoscaling.k8s.io/v1/fake" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/history" + "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/metrics" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/input/spec" "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" controllerfetcher "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/target/controller_fetcher" @@ -77,7 +78,46 @@ const ( testGcPeriod = time.Minute ) -func TestLoadPods(t *testing.T) { +// NewClusterState returns a new clusterState with no pods. +func NewFakeClusterState(vpas map[model.VpaID]*model.Vpa, pods map[model.PodID]*model.PodState) *fakeClusterState { + return &fakeClusterState{ + stubbedVPAs: vpas, + stubbedPods: pods, + addedSamples: make(map[model.ContainerID][]*model.ContainerUsageSampleWithKey), + } +} + +type fakeClusterState struct { + model.ClusterState + addedPods []model.PodID + addedSamples map[model.ContainerID][]*model.ContainerUsageSampleWithKey + stubbedVPAs map[model.VpaID]*model.Vpa + stubbedPods map[model.PodID]*model.PodState +} + +func (cs *fakeClusterState) AddSample(sample *model.ContainerUsageSampleWithKey) error { + samplesForContainer := cs.addedSamples[sample.Container] + cs.addedSamples[sample.Container] = append(samplesForContainer, sample) + return nil +} + +func (cs *fakeClusterState) AddOrUpdatePod(podID model.PodID, _ labels.Set, _ v1.PodPhase) { + cs.addedPods = append(cs.addedPods, podID) +} + +func (cs *fakeClusterState) Pods() map[model.PodID]*model.PodState { + return cs.stubbedPods +} + +func (cs *fakeClusterState) VPAs() map[model.VpaID]*model.Vpa { + return cs.stubbedVPAs +} + +func (cs *fakeClusterState) StateMapSize() int { + return 0 +} + +func TestLoadVPAs(t *testing.T) { type testCase struct { name string @@ -329,11 +369,11 @@ func TestLoadPods(t *testing.T) { } if !tc.expectedVpaFetch { - assert.NotContains(t, clusterState.Vpas, vpaID) + assert.NotContains(t, clusterState.VPAs(), vpaID) return } - assert.Contains(t, clusterState.Vpas, vpaID) - storedVpa := clusterState.Vpas[vpaID] + assert.Contains(t, clusterState.VPAs(), vpaID) + storedVpa := clusterState.VPAs()[vpaID] if tc.expectedSelector != nil { assert.NotNil(t, storedVpa.PodSelector) assert.Equal(t, tc.expectedSelector.String(), storedVpa.PodSelector.String()) @@ -380,7 +420,70 @@ func makeTestSpecClient(podLabels []map[string]string) spec.SpecClient { } } -func TestClusterStateFeeder_LoadPods(t *testing.T) { +func newTestContainerSpec(podID model.PodID, containerName string, millicores int, memory int64) spec.BasicContainerSpec { + containerID := model.ContainerID{ + PodID: podID, + ContainerName: containerName, + } + requestedResources := model.Resources{ + model.ResourceCPU: model.ResourceAmount(millicores), + model.ResourceMemory: model.ResourceAmount(memory), + } + return spec.BasicContainerSpec{ + ID: containerID, + Image: containerName + "Image", + Request: requestedResources, + } +} + +func newTestPodSpec(podId model.PodID, containerSpecs []spec.BasicContainerSpec, initContainerSpecs []spec.BasicContainerSpec) *spec.BasicPodSpec { + return &spec.BasicPodSpec{ + ID: podId, + PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, + Containers: containerSpecs, + InitContainers: initContainerSpecs, + } +} + +func TestClusterStateFeeder_LoadPods_ContainerTracking(t *testing.T) { + podWithoutInitContainersID := model.PodID{Namespace: "default", PodName: "PodWithoutInitContainers"} + containerSpecs := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithoutInitContainersID, "container1", 500, 512*1024*1024), + newTestContainerSpec(podWithoutInitContainersID, "container2", 1000, 1024*1024*1024), + } + podWithoutInitContainers := newTestPodSpec(podWithoutInitContainersID, containerSpecs, nil) + + podWithInitContainersID := model.PodID{Namespace: "default", PodName: "PodWithInitContainers"} + containerSpecs2 := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithInitContainersID, "container1", 2000, 2048*1024*1024), + } + initContainerSpecs2 := []spec.BasicContainerSpec{ + newTestContainerSpec(podWithInitContainersID, "init1", 40, 128*1024*1024), + newTestContainerSpec(podWithInitContainersID, "init2", 100, 256*1024*1024), + } + podWithInitContainers := newTestPodSpec(podWithInitContainersID, containerSpecs2, initContainerSpecs2) + + client := &testSpecClient{pods: []*spec.BasicPodSpec{podWithoutInitContainers, podWithInitContainers}} + + clusterState := model.NewClusterState(testGcPeriod) + + feeder := clusterStateFeeder{ + specClient: client, + memorySaveMode: false, + clusterState: clusterState, + } + + feeder.LoadPods() + + assert.Equal(t, len(feeder.clusterState.Pods()), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithInitContainersID].Containers), 1) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithInitContainersID].InitContainers), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithoutInitContainersID].Containers), 2) + assert.Equal(t, len(feeder.clusterState.Pods()[podWithoutInitContainersID].InitContainers), 0) + +} + +func TestClusterStateFeeder_LoadPods_MemorySaverMode(t *testing.T) { for _, tc := range []struct { Name string VPALabelSelectors []string @@ -427,14 +530,14 @@ func TestClusterStateFeeder_LoadPods(t *testing.T) { }, } { t.Run(tc.Name, func(t *testing.T) { - clusterState := model.NewClusterState(testGcPeriod) + vpas := make(map[model.VpaID]*model.Vpa) for i, selector := range tc.VPALabelSelectors { vpaLabel, err := labels.Parse(selector) assert.NoError(t, err) - clusterState.Vpas = map[model.VpaID]*model.Vpa{ - {VpaName: fmt.Sprintf("test-vpa-%d", i), Namespace: "default"}: {PodSelector: vpaLabel}, - } + key := model.VpaID{VpaName: fmt.Sprintf("test-vpa-%d", i), Namespace: "default"} + vpas[key] = &model.Vpa{PodSelector: vpaLabel} } + clusterState := NewFakeClusterState(vpas, nil) feeder := clusterStateFeeder{ specClient: makeTestSpecClient(tc.PodLabels), @@ -443,7 +546,9 @@ func TestClusterStateFeeder_LoadPods(t *testing.T) { } feeder.LoadPods() - assert.Len(t, feeder.clusterState.Pods, tc.TrackedPods, "number of pods is not %d", tc.TrackedPods) + assert.Len(t, clusterState.addedPods, tc.TrackedPods, "number of pods is not %d", tc.TrackedPods) + + clusterState = NewFakeClusterState(vpas, nil) feeder = clusterStateFeeder{ specClient: makeTestSpecClient(tc.PodLabels), @@ -452,11 +557,100 @@ func TestClusterStateFeeder_LoadPods(t *testing.T) { } feeder.LoadPods() - assert.Len(t, feeder.clusterState.Pods, len(tc.PodLabels), "number of pods is not %d", len(tc.PodLabels)) + assert.Len(t, clusterState.addedPods, len(tc.PodLabels), "number of pods is not %d", len(tc.PodLabels)) }) } } +func newContainerMetricsSnapshot(id model.ContainerID, cpuUsage int64, memUsage int64) (*metrics.ContainerMetricsSnapshot, []*model.ContainerUsageSampleWithKey) { + snapshotTimestamp := time.Now() + snapshotWindow := time.Duration(1234) + snapshot := &metrics.ContainerMetricsSnapshot{ + ID: id, + SnapshotTime: snapshotTimestamp, + SnapshotWindow: snapshotWindow, + Usage: model.Resources{ + model.ResourceCPU: model.ResourceAmount(cpuUsage), + model.ResourceMemory: model.ResourceAmount(memUsage), + }, + } + samples := []*model.ContainerUsageSampleWithKey{ + { + Container: id, + ContainerUsageSample: model.ContainerUsageSample{ + MeasureStart: snapshotTimestamp, + Resource: model.ResourceCPU, + Usage: model.ResourceAmount(cpuUsage), + }, + }, + { + Container: id, + ContainerUsageSample: model.ContainerUsageSample{ + MeasureStart: snapshotTimestamp, + Resource: model.ResourceMemory, + Usage: model.ResourceAmount(memUsage), + }, + }, + } + return snapshot, samples +} + +type fakeMetricsClient struct { + snapshots []*metrics.ContainerMetricsSnapshot +} + +func (m fakeMetricsClient) GetContainersMetrics() ([]*metrics.ContainerMetricsSnapshot, error) { + return m.snapshots, nil +} + +func TestClusterStateFeeder_LoadRealTimeMetrics(t *testing.T) { + namespaceName := "test-namespace" + podID := model.PodID{Namespace: namespaceName, PodName: "Pod"} + regularContainer1 := model.ContainerID{PodID: podID, ContainerName: "Container1"} + regularContainer2 := model.ContainerID{PodID: podID, ContainerName: "Container2"} + initContainer := model.ContainerID{PodID: podID, ContainerName: "InitContainer"} + + pods := map[model.PodID]*model.PodState{ + podID: {ID: podID, + Containers: map[string]*model.ContainerState{ + "Container1": {}, + "Container2": {}, + }, + InitContainers: []string{ + "InitContainer", + }}, + } + + var containerMetricsSnapshots []*metrics.ContainerMetricsSnapshot + + regularContainer1MetricsSnapshot, regularContainer1UsageSamples := newContainerMetricsSnapshot(regularContainer1, 100, 1024) + containerMetricsSnapshots = append(containerMetricsSnapshots, regularContainer1MetricsSnapshot) + regularContainer2MetricsSnapshot, regularContainer2UsageSamples := newContainerMetricsSnapshot(regularContainer2, 200, 2048) + containerMetricsSnapshots = append(containerMetricsSnapshots, regularContainer2MetricsSnapshot) + initContainer1MetricsSnapshots, _ := newContainerMetricsSnapshot(initContainer, 300, 3072) + containerMetricsSnapshots = append(containerMetricsSnapshots, initContainer1MetricsSnapshots) + + clusterState := NewFakeClusterState(nil, pods) + + feeder := clusterStateFeeder{ + memorySaveMode: false, + clusterState: clusterState, + metricsClient: fakeMetricsClient{snapshots: containerMetricsSnapshots}, + } + + feeder.LoadRealTimeMetrics() + + assert.Equal(t, 2, len(clusterState.addedSamples)) + + samplesForContainer1 := clusterState.addedSamples[regularContainer1] + assert.Contains(t, samplesForContainer1, regularContainer1UsageSamples[0]) + assert.Contains(t, samplesForContainer1, regularContainer1UsageSamples[1]) + + samplesForContainer2 := clusterState.addedSamples[regularContainer2] + assert.Contains(t, samplesForContainer2, regularContainer2UsageSamples[0]) + assert.Contains(t, samplesForContainer2, regularContainer2UsageSamples[1]) +} + type fakeHistoryProvider struct { history map[model.PodID]*history.PodHistory err error @@ -506,10 +700,10 @@ func TestClusterStateFeeder_InitFromHistoryProvider(t *testing.T) { clusterState: clusterState, } feeder.InitFromHistoryProvider(&provider) - if !assert.Contains(t, feeder.clusterState.Pods, pod1) { + if !assert.Contains(t, feeder.clusterState.Pods(), pod1) { return } - pod1State := feeder.clusterState.Pods[pod1] + pod1State := feeder.clusterState.Pods()[pod1] if !assert.Contains(t, pod1State.Containers, containerCpu) { return } @@ -700,7 +894,7 @@ func TestCanCleanupCheckpoints(t *testing.T) { coreClient: client.CoreV1(), vpaLister: vpaLister, vpaCheckpointClient: checkpointClient, - clusterState: &model.ClusterState{}, + clusterState: model.NewClusterState(testGcPeriod), recommenderName: "default", } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go b/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go index 0db773fc52c..968765b9171 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/history/history_provider.go @@ -75,7 +75,7 @@ func newEmptyHistory() *PodHistory { // HistoryProvider gives history of all pods in a cluster. // TODO(schylek): this interface imposes how history is represented which doesn't work well with checkpoints. -// Consider refactoring to passing ClusterState and create history provider working with checkpoints. +// Consider refactoring to passing clusterState and create history provider working with checkpoints. type HistoryProvider interface { GetClusterHistory() (map[model.PodID]*PodHistory, error) } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go b/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go index 7f8905e3f73..f86dcc17d8f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/metrics/metrics_source.go @@ -58,7 +58,7 @@ func (s podMetricsSource) List(ctx context.Context, namespace string, opts v1.Li type externalMetricsClient struct { externalClient external_metrics.ExternalMetricsClient options ExternalClientOptions - clusterState *model.ClusterState + clusterState model.ClusterState } // ExternalClientOptions specifies parameters for using an External Metrics Client. @@ -69,7 +69,7 @@ type ExternalClientOptions struct { } // NewExternalClient returns a Source for an External Metrics Client. -func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options ExternalClientOptions) PodMetricsLister { +func NewExternalClient(c *rest.Config, clusterState model.ClusterState, options ExternalClientOptions) PodMetricsLister { extClient, err := external_metrics.NewForConfig(c) if err != nil { klog.ErrorS(err, "Failed initializing external metrics client") @@ -85,7 +85,7 @@ func NewExternalClient(c *rest.Config, clusterState *model.ClusterState, options func (s *externalMetricsClient) List(ctx context.Context, namespace string, opts v1.ListOptions) (*v1beta1.PodMetricsList, error) { result := v1beta1.PodMetricsList{} - for _, vpa := range s.clusterState.Vpas { + for _, vpa := range s.clusterState.VPAs() { if vpa.PodCount == 0 { continue } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go index dba046b871a..81de7307358 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client.go @@ -32,6 +32,8 @@ type BasicPodSpec struct { PodLabels map[string]string // List of containers within this pod. Containers []BasicContainerSpec + // List of init containers within this pod. + InitContainers []BasicContainerSpec // PodPhase describing current life cycle phase of the Pod. Phase v1.PodPhase } @@ -82,21 +84,23 @@ func newBasicPodSpec(pod *v1.Pod) *BasicPodSpec { PodName: pod.Name, Namespace: pod.Namespace, } - containerSpecs := newContainerSpecs(podId, pod) + containerSpecs := newContainerSpecs(podId, pod.Spec.Containers) + initContainerSpecs := newContainerSpecs(podId, pod.Spec.InitContainers) basicPodSpec := &BasicPodSpec{ - ID: podId, - PodLabels: pod.Labels, - Containers: containerSpecs, - Phase: pod.Status.Phase, + ID: podId, + PodLabels: pod.Labels, + Containers: containerSpecs, + InitContainers: initContainerSpecs, + Phase: pod.Status.Phase, } return basicPodSpec } -func newContainerSpecs(podID model.PodID, pod *v1.Pod) []BasicContainerSpec { +func newContainerSpecs(podID model.PodID, containers []v1.Container) []BasicContainerSpec { var containerSpecs []BasicContainerSpec - for _, container := range pod.Spec.Containers { + for _, container := range containers { containerSpec := newContainerSpec(podID, container) containerSpecs = append(containerSpecs, containerSpec) } diff --git a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go index 4a2a212e498..e6cfc606ab1 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go @@ -82,6 +82,13 @@ spec: requests: memory: "4096Mi" cpu: "4000m" + initContainers: + - name: Name21-init + image: Name21-initImage + resources: + requests: + memory: "128Mi" + cpu: "40m" ` type podListerMock struct { @@ -116,8 +123,10 @@ func newSpecClientTestCase() *specClientTestCase { containerSpec21 := newTestContainerSpec(podID2, "Name21", 2000, 2048*1024*1024) containerSpec22 := newTestContainerSpec(podID2, "Name22", 4000, 4096*1024*1024) - podSpec1 := newTestPodSpec(podID1, containerSpec11, containerSpec12) - podSpec2 := newTestPodSpec(podID2, containerSpec21, containerSpec22) + initContainerSpec21 := newTestContainerSpec(podID2, "Name21-init", 40, 128*1024*1024) + + podSpec1 := newTestPodSpec(podID1, []BasicContainerSpec{containerSpec11, containerSpec12}, nil) + podSpec2 := newTestPodSpec(podID2, []BasicContainerSpec{containerSpec21, containerSpec22}, []BasicContainerSpec{initContainerSpec21}) return &specClientTestCase{ podSpecs: []*BasicPodSpec{podSpec1, podSpec2}, @@ -141,11 +150,12 @@ func newTestContainerSpec(podID model.PodID, containerName string, milicores int } } -func newTestPodSpec(podId model.PodID, containerSpecs ...BasicContainerSpec) *BasicPodSpec { +func newTestPodSpec(podId model.PodID, containerSpecs []BasicContainerSpec, initContainerSpecs []BasicContainerSpec) *BasicPodSpec { return &BasicPodSpec{ - ID: podId, - PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, - Containers: containerSpecs, + ID: podId, + PodLabels: map[string]string{podId.PodName + "LabelKey": podId.PodName + "LabelValue"}, + Containers: containerSpecs, + InitContainers: initContainerSpecs, } } diff --git a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go index c45025bab5d..c18e860f4ff 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state.go @@ -307,12 +307,12 @@ func AggregateStateByContainerName(aggregateContainerStateMap aggregateContainer // present in the cluster state. type ContainerStateAggregatorProxy struct { containerID ContainerID - cluster *ClusterState + cluster *clusterState } // NewContainerStateAggregatorProxy creates a ContainerStateAggregatorProxy // pointing to the cluster state. -func NewContainerStateAggregatorProxy(cluster *ClusterState, containerID ContainerID) ContainerStateAggregator { +func NewContainerStateAggregatorProxy(cluster *clusterState, containerID ContainerID) ContainerStateAggregator { return &ContainerStateAggregatorProxy{containerID, cluster} } diff --git a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go index 0886bd5db9b..fd0e2f79a7f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/aggregate_container_state_test.go @@ -38,7 +38,7 @@ var ( } ) -func addTestCPUSample(cluster *ClusterState, container ContainerID, cpuCores float64) error { +func addTestCPUSample(cluster ClusterState, container ContainerID, cpuCores float64) error { sample := ContainerUsageSampleWithKey{ Container: container, ContainerUsageSample: ContainerUsageSample{ @@ -50,7 +50,7 @@ func addTestCPUSample(cluster *ClusterState, container ContainerID, cpuCores flo return cluster.AddSample(&sample) } -func addTestMemorySample(cluster *ClusterState, container ContainerID, memoryBytes float64) error { +func addTestMemorySample(cluster ClusterState, container ContainerID, memoryBytes float64) error { sample := ContainerUsageSampleWithKey{ Container: container, ContainerUsageSample: ContainerUsageSample{ diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go index 8fd2fd8f030..be8eceaed86 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster.go @@ -40,17 +40,39 @@ const ( // VPA objects), aggregated utilization of compute resources (CPU, memory) and // events (container OOMs). // All input to the VPA Recommender algorithm lives in this structure. -type ClusterState struct { +type ClusterState interface { + StateMapSize() int + AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) + GetContainer(containerID ContainerID) *ContainerState + DeletePod(podID PodID) + AddOrUpdateContainer(containerID ContainerID, request Resources) error + AddSample(sample *ContainerUsageSampleWithKey) error + RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error + AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error + DeleteVpa(vpaID VpaID) error + MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey + RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) + RecordRecommendation(vpa *Vpa, now time.Time) error + GetMatchingPods(vpa *Vpa) []PodID + GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion + GetControllingVPA(pod *PodState) *Vpa + VPAs() map[VpaID]*Vpa + SetObservedVPAs([]*vpa_types.VerticalPodAutoscaler) + ObservedVPAs() []*vpa_types.VerticalPodAutoscaler + Pods() map[PodID]*PodState +} + +type clusterState struct { // Pods in the cluster. - Pods map[PodID]*PodState + pods map[PodID]*PodState // VPA objects in the cluster. - Vpas map[VpaID]*Vpa + vpas map[VpaID]*Vpa // VPA objects in the cluster that have no recommendation mapped to the first // time we've noticed the recommendation missing or last time we logged // a warning about it. - EmptyVPAs map[VpaID]time.Time + emptyVPAs map[VpaID]time.Time // Observed VPAs. Used to check if there are updates needed. - ObservedVpas []*vpa_types.VerticalPodAutoscaler + observedVPAs []*vpa_types.VerticalPodAutoscaler // All container aggregations where the usage samples are stored. aggregateStateMap aggregateContainerStatesMap @@ -63,7 +85,7 @@ type ClusterState struct { } // StateMapSize is the number of pods being tracked by the VPA -func (cluster *ClusterState) StateMapSize() int { +func (cluster *clusterState) StateMapSize() int { return len(cluster.aggregateStateMap) } @@ -93,16 +115,18 @@ type PodState struct { labelSetKey labelSetKey // Containers that belong to the Pod, keyed by the container name. Containers map[string]*ContainerState + // InitContainers is a list of init containers names which belong to the Pod. + InitContainers []string // PodPhase describing current life cycle phase of the Pod. Phase apiv1.PodPhase } -// NewClusterState returns a new ClusterState with no pods. -func NewClusterState(gcInterval time.Duration) *ClusterState { - return &ClusterState{ - Pods: make(map[PodID]*PodState), - Vpas: make(map[VpaID]*Vpa), - EmptyVPAs: make(map[VpaID]time.Time), +// NewClusterState returns a new clusterState with no pods. +func NewClusterState(gcInterval time.Duration) *clusterState { + return &clusterState{ + pods: make(map[PodID]*PodState), + vpas: make(map[VpaID]*Vpa), + emptyVPAs: make(map[VpaID]time.Time), aggregateStateMap: make(aggregateContainerStatesMap), labelSetMap: make(labelSetMap), lastAggregateContainerStateGC: time.Unix(0, 0), @@ -122,11 +146,11 @@ type ContainerUsageSampleWithKey struct { // the Cluster object. // If the labels of the pod have changed, it updates the links between the containers // and the aggregations. -func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) { - pod, podExists := cluster.Pods[podID] +func (cluster *clusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, phase apiv1.PodPhase) { + pod, podExists := cluster.pods[podID] if !podExists { pod = newPod(podID) - cluster.Pods[podID] = pod + cluster.pods[podID] = pod } newlabelSetKey := cluster.getLabelSetKey(newLabels) @@ -149,8 +173,8 @@ func (cluster *ClusterState) AddOrUpdatePod(podID PodID, newLabels labels.Set, p // addPodToItsVpa increases the count of Pods associated with a VPA object. // Does a scan similar to findOrCreateAggregateContainerState so could be optimized if needed. -func (cluster *ClusterState) addPodToItsVpa(pod *PodState) { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) addPodToItsVpa(pod *PodState) { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { vpa.PodCount++ } @@ -158,8 +182,8 @@ func (cluster *ClusterState) addPodToItsVpa(pod *PodState) { } // removePodFromItsVpa decreases the count of Pods associated with a VPA object. -func (cluster *ClusterState) removePodFromItsVpa(pod *PodState) { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) removePodFromItsVpa(pod *PodState) { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { vpa.PodCount-- } @@ -168,8 +192,8 @@ func (cluster *ClusterState) removePodFromItsVpa(pod *PodState) { // GetContainer returns the ContainerState object for a given ContainerID or // null if it's not present in the model. -func (cluster *ClusterState) GetContainer(containerID ContainerID) *ContainerState { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) GetContainer(containerID ContainerID) *ContainerState { + pod, podExists := cluster.pods[containerID.PodID] if podExists { container, containerExists := pod.Containers[containerID.ContainerName] if containerExists { @@ -180,20 +204,20 @@ func (cluster *ClusterState) GetContainer(containerID ContainerID) *ContainerSta } // DeletePod removes an existing pod from the cluster. -func (cluster *ClusterState) DeletePod(podID PodID) { - pod, found := cluster.Pods[podID] +func (cluster *clusterState) DeletePod(podID PodID) { + pod, found := cluster.pods[podID] if found { cluster.removePodFromItsVpa(pod) } - delete(cluster.Pods, podID) + delete(cluster.pods, podID) } // AddOrUpdateContainer creates a new container with the given ContainerID and -// adds it to the parent pod in the ClusterState object, if not yet present. -// Requires the pod to be added to the ClusterState first. Otherwise an error is +// adds it to the parent pod in the clusterState object, if not yet present. +// Requires the pod to be added to the clusterState first. Otherwise an error is // returned. -func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID, request Resources) error { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) AddOrUpdateContainer(containerID ContainerID, request Resources) error { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { return NewKeyError(containerID.PodID) } @@ -207,11 +231,11 @@ func (cluster *ClusterState) AddOrUpdateContainer(containerID ContainerID, reque return nil } -// AddSample adds a new usage sample to the proper container in the ClusterState +// AddSample adds a new usage sample to the proper container in the clusterState // object. Requires the container as well as the parent pod to be added to the -// ClusterState first. Otherwise an error is returned. -func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) error { - pod, podExists := cluster.Pods[sample.Container.PodID] +// clusterState first. Otherwise an error is returned. +func (cluster *clusterState) AddSample(sample *ContainerUsageSampleWithKey) error { + pod, podExists := cluster.pods[sample.Container.PodID] if !podExists { return NewKeyError(sample.Container.PodID) } @@ -226,8 +250,8 @@ func (cluster *ClusterState) AddSample(sample *ContainerUsageSampleWithKey) erro } // RecordOOM adds info regarding OOM event in the model as an artificial memory sample. -func (cluster *ClusterState) RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error { - pod, podExists := cluster.Pods[containerID.PodID] +func (cluster *clusterState) RecordOOM(containerID ContainerID, timestamp time.Time, requestedMemory ResourceAmount) error { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { return NewKeyError(containerID.PodID) } @@ -242,11 +266,11 @@ func (cluster *ClusterState) RecordOOM(containerID ContainerID, timestamp time.T return nil } -// AddOrUpdateVpa adds a new VPA with a given ID to the ClusterState if it +// AddOrUpdateVpa adds a new VPA with a given ID to the clusterState if it // didn't yet exist. If the VPA already existed but had a different pod // selector, the pod selector is updated. Updates the links between the VPA and // all aggregations it matches. -func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error { +func (cluster *clusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAutoscaler, selector labels.Selector) error { vpaID := VpaID{Namespace: apiObject.Namespace, VpaName: apiObject.Name} annotationsMap := apiObject.Annotations conditionsMap := make(vpaConditionsMap) @@ -258,7 +282,7 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto currentRecommendation = apiObject.Status.Recommendation } - vpa, vpaExists := cluster.Vpas[vpaID] + vpa, vpaExists := cluster.vpas[vpaID] if vpaExists && (vpa.PodSelector.String() != selector.String()) { // Pod selector was changed. Delete the VPA object and recreate // it with the new selector. @@ -269,7 +293,7 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto } if !vpaExists { vpa = NewVpa(vpaID, selector, apiObject.CreationTimestamp.Time) - cluster.Vpas[vpaID] = vpa + cluster.vpas[vpaID] = vpa for aggregationKey, aggregation := range cluster.aggregateStateMap { vpa.UseAggregationIfMatching(aggregationKey, aggregation) } @@ -285,20 +309,36 @@ func (cluster *ClusterState) AddOrUpdateVpa(apiObject *vpa_types.VerticalPodAuto return nil } -// DeleteVpa removes a VPA with the given ID from the ClusterState. -func (cluster *ClusterState) DeleteVpa(vpaID VpaID) error { - vpa, vpaExists := cluster.Vpas[vpaID] +// DeleteVpa removes a VPA with the given ID from the clusterState. +func (cluster *clusterState) DeleteVpa(vpaID VpaID) error { + vpa, vpaExists := cluster.vpas[vpaID] if !vpaExists { return NewKeyError(vpaID) } for _, state := range vpa.aggregateContainerStates { state.MarkNotAutoscaled() } - delete(cluster.Vpas, vpaID) - delete(cluster.EmptyVPAs, vpaID) + delete(cluster.vpas, vpaID) + delete(cluster.emptyVPAs, vpaID) return nil } +func (cluster *clusterState) VPAs() map[VpaID]*Vpa { + return cluster.vpas +} + +func (cluster *clusterState) Pods() map[PodID]*PodState { + return cluster.pods +} + +func (cluster *clusterState) SetObservedVPAs(observedVPAs []*vpa_types.VerticalPodAutoscaler) { + cluster.observedVPAs = observedVPAs +} + +func (cluster *clusterState) ObservedVPAs() []*vpa_types.VerticalPodAutoscaler { + return cluster.observedVPAs +} + func newPod(id PodID) *PodState { return &PodState{ ID: id, @@ -308,7 +348,7 @@ func newPod(id PodID) *PodState { // getLabelSetKey puts the given labelSet in the global labelSet map and returns a // corresponding labelSetKey. -func (cluster *ClusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { +func (cluster *clusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { labelSetKey := labelSetKey(labelSet.String()) cluster.labelSetMap[labelSetKey] = labelSet return labelSetKey @@ -316,7 +356,7 @@ func (cluster *ClusterState) getLabelSetKey(labelSet labels.Set) labelSetKey { // MakeAggregateStateKey returns the AggregateStateKey that should be used // to aggregate usage samples from a container with the given name in a given pod. -func (cluster *ClusterState) MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey { +func (cluster *clusterState) MakeAggregateStateKey(pod *PodState, containerName string) AggregateStateKey { return aggregateStateKey{ namespace: pod.ID.Namespace, containerName: containerName, @@ -326,9 +366,9 @@ func (cluster *ClusterState) MakeAggregateStateKey(pod *PodState, containerName } // aggregateStateKeyForContainerID returns the AggregateStateKey for the ContainerID. -// The pod with the corresponding PodID must already be present in the ClusterState. -func (cluster *ClusterState) aggregateStateKeyForContainerID(containerID ContainerID) AggregateStateKey { - pod, podExists := cluster.Pods[containerID.PodID] +// The pod with the corresponding PodID must already be present in the clusterState. +func (cluster *clusterState) aggregateStateKeyForContainerID(containerID ContainerID) AggregateStateKey { + pod, podExists := cluster.pods[containerID.PodID] if !podExists { panic(fmt.Sprintf("Pod not present in the ClusterState: %s/%s", containerID.PodID.Namespace, containerID.PodID.PodName)) } @@ -337,22 +377,22 @@ func (cluster *ClusterState) aggregateStateKeyForContainerID(containerID Contain // findOrCreateAggregateContainerState returns (possibly newly created) AggregateContainerState // that should be used to aggregate usage samples from container with a given ID. -// The pod with the corresponding PodID must already be present in the ClusterState. -func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID ContainerID) *AggregateContainerState { +// The pod with the corresponding PodID must already be present in the clusterState. +func (cluster *clusterState) findOrCreateAggregateContainerState(containerID ContainerID) *AggregateContainerState { aggregateStateKey := cluster.aggregateStateKeyForContainerID(containerID) aggregateContainerState, aggregateStateExists := cluster.aggregateStateMap[aggregateStateKey] if !aggregateStateExists { aggregateContainerState = NewAggregateContainerState() cluster.aggregateStateMap[aggregateStateKey] = aggregateContainerState // Link the new aggregation to the existing VPAs. - for _, vpa := range cluster.Vpas { + for _, vpa := range cluster.vpas { vpa.UseAggregationIfMatching(aggregateStateKey, aggregateContainerState) } } return aggregateContainerState } -// garbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// garbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the clusterState. // AggregateCollectionState is obsolete in following situations: // 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: // @@ -361,7 +401,7 @@ func (cluster *ClusterState) findOrCreateAggregateContainerState(containerID Con // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *clusterState) garbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { klog.V(1).InfoS("Garbage collection of AggregateCollectionStates triggered") keysToDelete := make([]AggregateStateKey, 0) contributiveKeys := cluster.getContributiveAggregateStateKeys(ctx, controllerFetcher) @@ -379,13 +419,13 @@ func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context } for _, key := range keysToDelete { delete(cluster.aggregateStateMap, key) - for _, vpa := range cluster.Vpas { + for _, vpa := range cluster.vpas { vpa.DeleteAggregation(key) } } } -// RateLimitedGarbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the ClusterState. +// RateLimitedGarbageCollectAggregateCollectionStates removes obsolete AggregateCollectionStates from the clusterState. // It performs clean up only if more than `gcInterval` passed since the last time it performed a cleanup. // AggregateCollectionState is obsolete in following situations: // 1) It has no samples and there are no more contributive pods - a pod is contributive in any of following situations: @@ -395,7 +435,7 @@ func (cluster *ClusterState) garbageCollectAggregateCollectionStates(ctx context // // 2) The last sample is too old to give meaningful recommendation (>8 days), // 3) There are no samples and the aggregate state was created >8 days ago. -func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { +func (cluster *clusterState) RateLimitedGarbageCollectAggregateCollectionStates(ctx context.Context, now time.Time, controllerFetcher controllerfetcher.ControllerFetcher) { if now.Sub(cluster.lastAggregateContainerStateGC) < cluster.gcInterval { return } @@ -403,9 +443,9 @@ func (cluster *ClusterState) RateLimitedGarbageCollectAggregateCollectionStates( cluster.lastAggregateContainerStateGC = now } -func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[AggregateStateKey]bool { +func (cluster *clusterState) getContributiveAggregateStateKeys(ctx context.Context, controllerFetcher controllerfetcher.ControllerFetcher) map[AggregateStateKey]bool { contributiveKeys := map[AggregateStateKey]bool{} - for _, pod := range cluster.Pods { + for _, pod := range cluster.pods { // Pod is considered contributive in any of following situations: // 1) It is in active state - i.e. not PodSucceeded nor PodFailed. // 2) Its associated controller (e.g. Deployment) still exists. @@ -423,17 +463,17 @@ func (cluster *ClusterState) getContributiveAggregateStateKeys(ctx context.Conte // RecordRecommendation marks the state of recommendation in the cluster. We // keep track of empty recommendations and log information about them // periodically. -func (cluster *ClusterState) RecordRecommendation(vpa *Vpa, now time.Time) error { +func (cluster *clusterState) RecordRecommendation(vpa *Vpa, now time.Time) error { if vpa.Recommendation != nil && len(vpa.Recommendation.ContainerRecommendations) > 0 { - delete(cluster.EmptyVPAs, vpa.ID) + delete(cluster.emptyVPAs, vpa.ID) return nil } - lastLogged, ok := cluster.EmptyVPAs[vpa.ID] + lastLogged, ok := cluster.emptyVPAs[vpa.ID] if !ok { - cluster.EmptyVPAs[vpa.ID] = now + cluster.emptyVPAs[vpa.ID] = now } else { if lastLogged.Add(RecommendationMissingMaxDuration).Before(now) { - cluster.EmptyVPAs[vpa.ID] = now + cluster.emptyVPAs[vpa.ID] = now return fmt.Errorf("VPA %s/%s is missing recommendation for more than %v", vpa.ID.Namespace, vpa.ID.VpaName, RecommendationMissingMaxDuration) } } @@ -442,9 +482,9 @@ func (cluster *ClusterState) RecordRecommendation(vpa *Vpa, now time.Time) error // GetMatchingPods returns a list of currently active pods that match the // given VPA. Traverses through all pods in the cluster - use sparingly. -func (cluster *ClusterState) GetMatchingPods(vpa *Vpa) []PodID { +func (cluster *clusterState) GetMatchingPods(vpa *Vpa) []PodID { matchingPods := []PodID{} - for podID, pod := range cluster.Pods { + for podID, pod := range cluster.pods { if vpa_utils.PodLabelsMatchVPA(podID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { matchingPods = append(matchingPods, podID) @@ -454,7 +494,7 @@ func (cluster *ClusterState) GetMatchingPods(vpa *Vpa) []PodID { } // GetControllerForPodUnderVPA returns controller associated with given Pod. Returns nil if Pod is not controlled by a VPA object. -func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { +func (cluster *clusterState) GetControllerForPodUnderVPA(ctx context.Context, pod *PodState, controllerFetcher controllerfetcher.ControllerFetcher) *controllerfetcher.ControllerKeyWithAPIVersion { controllingVPA := cluster.GetControllingVPA(pod) if controllingVPA != nil { controller := &controllerfetcher.ControllerKeyWithAPIVersion{ @@ -472,8 +512,8 @@ func (cluster *ClusterState) GetControllerForPodUnderVPA(ctx context.Context, po } // GetControllingVPA returns a VPA object controlling given Pod. -func (cluster *ClusterState) GetControllingVPA(pod *PodState) *Vpa { - for _, vpa := range cluster.Vpas { +func (cluster *clusterState) GetControllingVPA(pod *PodState) *Vpa { + for _, vpa := range cluster.vpas { if vpa_utils.PodLabelsMatchVPA(pod.ID.Namespace, cluster.labelSetMap[pod.labelSetKey], vpa.ID.Namespace, vpa.PodSelector) { return vpa diff --git a/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go b/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go index e046b95af3c..c0e67a6524f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/model/cluster_test.go @@ -93,7 +93,7 @@ func TestClusterAddSample(t *testing.T) { assert.NoError(t, cluster.AddSample(makeTestUsageSample())) // Verify that the sample was aggregated into the container stats. - containerStats := cluster.Pods[testPodID].Containers["container-1"] + containerStats := cluster.pods[testPodID].Containers["container-1"] assert.Equal(t, testTimestamp, containerStats.LastCPUSampleStart) } @@ -180,7 +180,7 @@ func TestClusterGCAggregateContainerStateDeletesEmptyInactiveWithoutController(t assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, vpa.aggregateContainerStates) - cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.pods[pod.ID].Phase = apiv1.PodSucceeded cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggregateContainerState should be empty as the pod is no longer active, controller is not alive @@ -211,7 +211,7 @@ func TestClusterGCAggregateContainerStateLeavesEmptyInactiveWithController(t *te assert.NotEmpty(t, cluster.aggregateStateMap) assert.NotEmpty(t, vpa.aggregateContainerStates) - cluster.Pods[pod.ID].Phase = apiv1.PodSucceeded + cluster.pods[pod.ID].Phase = apiv1.PodSucceeded cluster.garbageCollectAggregateCollectionStates(ctx, testTimestamp, controller) // AggregateContainerState should not be deleted as the controller is still alive. @@ -342,13 +342,13 @@ func TestMissingKeys(t *testing.T) { assert.EqualError(t, err, "KeyError: {namespace-1 pod-1}") } -func addVpa(cluster *ClusterState, id VpaID, annotations vpaAnnotationsMap, selector string, targetRef *autoscaling.CrossVersionObjectReference) *Vpa { +func addVpa(cluster ClusterState, id VpaID, annotations vpaAnnotationsMap, selector string, targetRef *autoscaling.CrossVersionObjectReference) *Vpa { apiObject := test.VerticalPodAutoscaler().WithNamespace(id.Namespace). WithName(id.VpaName).WithContainer(testContainerID.ContainerName).WithAnnotations(annotations).WithTargetRef(targetRef).Get() return addVpaObject(cluster, id, apiObject, selector) } -func addVpaObject(cluster *ClusterState, id VpaID, vpa *vpa_types.VerticalPodAutoscaler, selector string) *Vpa { +func addVpaObject(cluster ClusterState, id VpaID, vpa *vpa_types.VerticalPodAutoscaler, selector string) *Vpa { labelSelector, _ := metav1.ParseToLabelSelector(selector) parsedSelector, _ := metav1.LabelSelectorAsSelector(labelSelector) err := cluster.AddOrUpdateVpa(vpa, parsedSelector) @@ -356,19 +356,19 @@ func addVpaObject(cluster *ClusterState, id VpaID, vpa *vpa_types.VerticalPodAut klog.ErrorS(err, "AddOrUpdateVpa() failed") os.Exit(255) } - return cluster.Vpas[id] + return cluster.VPAs()[id] } -func addTestVpa(cluster *ClusterState) *Vpa { +func addTestVpa(cluster ClusterState) *Vpa { return addVpa(cluster, testVpaID, testAnnotations, testSelectorStr, testTargetRef) } -func addTestPod(cluster *ClusterState) *PodState { +func addTestPod(cluster ClusterState) *PodState { cluster.AddOrUpdatePod(testPodID, testLabels, apiv1.PodRunning) - return cluster.Pods[testPodID] + return cluster.Pods()[testPodID] } -func addTestContainer(t *testing.T, cluster *ClusterState) *ContainerState { +func addTestContainer(t *testing.T, cluster ClusterState) *ContainerState { err := cluster.AddOrUpdateContainer(testContainerID, testRequest) assert.NoError(t, err) return cluster.GetContainer(testContainerID) @@ -612,7 +612,7 @@ func TestAddOrUpdateVPAPolicies(t *testing.T) { addTestContainer(t, cluster) if tc.oldVpa != nil { oldVpa := addVpaObject(cluster, testVpaID, tc.oldVpa, testSelectorStr) - if !assert.Contains(t, cluster.Vpas, testVpaID) { + if !assert.Contains(t, cluster.vpas, testVpaID) { t.FailNow() } assert.Len(t, oldVpa.aggregateContainerStates, 1, "Expected one container aggregation in VPA %v", testVpaID) @@ -622,7 +622,7 @@ func TestAddOrUpdateVPAPolicies(t *testing.T) { } tc.newVpa.Spec.ResourcePolicy = tc.resourcePolicy addVpaObject(cluster, testVpaID, tc.newVpa, testSelectorStr) - vpa, found := cluster.Vpas[testVpaID] + vpa, found := cluster.vpas[testVpaID] if !assert.True(t, found, "VPA %+v not found in cluster state.", testVpaID) { t.FailNow() } @@ -762,9 +762,9 @@ func TestRecordRecommendation(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cluster := NewClusterState(testGcPeriod) vpa := addVpa(cluster, testVpaID, testAnnotations, testSelectorStr, testTargetRef) - cluster.Vpas[testVpaID].Recommendation = tc.recommendation + cluster.vpas[testVpaID].Recommendation = tc.recommendation if !tc.lastLogged.IsZero() { - cluster.EmptyVPAs[testVpaID] = tc.lastLogged + cluster.emptyVPAs[testVpaID] = tc.lastLogged } err := cluster.RecordRecommendation(vpa, tc.now) @@ -773,10 +773,10 @@ func TestRecordRecommendation(t *testing.T) { } else { assert.NoError(t, err) if tc.expectedEmpty { - assert.Contains(t, cluster.EmptyVPAs, testVpaID) - assert.Equal(t, cluster.EmptyVPAs[testVpaID], tc.expectedLastLogged) + assert.Contains(t, cluster.emptyVPAs, testVpaID) + assert.Equal(t, cluster.emptyVPAs[testVpaID], tc.expectedLastLogged) } else { - assert.NotContains(t, cluster.EmptyVPAs, testVpaID) + assert.NotContains(t, cluster.emptyVPAs, testVpaID) } } }) @@ -924,7 +924,7 @@ func TestVPAWithMatchingPods(t *testing.T) { containerID := ContainerID{testPodID, "foo"} assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) } - assert.Equal(t, tc.expectedMatch, cluster.Vpas[vpa.ID].PodCount) + assert.Equal(t, tc.expectedMatch, cluster.vpas[vpa.ID].PodCount) }) } // Run with adding Pods first @@ -937,7 +937,7 @@ func TestVPAWithMatchingPods(t *testing.T) { assert.NoError(t, cluster.AddOrUpdateContainer(containerID, testRequest)) } vpa := addVpa(cluster, testVpaID, testAnnotations, tc.vpaSelector, testTargetRef) - assert.Equal(t, tc.expectedMatch, cluster.Vpas[vpa.ID].PodCount) + assert.Equal(t, tc.expectedMatch, cluster.vpas[vpa.ID].PodCount) }) } } diff --git a/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go b/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go index 1f456a80c18..273a2093f3f 100644 --- a/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go +++ b/vertical-pod-autoscaler/pkg/recommender/routines/recommender.go @@ -43,7 +43,7 @@ type Recommender interface { // RunOnce performs one iteration of recommender duties followed by update of recommendations in VPA objects. RunOnce() // GetClusterState returns ClusterState used by Recommender - GetClusterState() *model.ClusterState + GetClusterState() model.ClusterState // GetClusterStateFeeder returns ClusterStateFeeder used by Recommender GetClusterStateFeeder() input.ClusterStateFeeder // UpdateVPAs computes recommendations and sends VPAs status updates to API Server @@ -55,7 +55,7 @@ type Recommender interface { } type recommender struct { - clusterState *model.ClusterState + clusterState model.ClusterState clusterStateFeeder input.ClusterStateFeeder checkpointWriter checkpoint.CheckpointWriter checkpointsGCInterval time.Duration @@ -68,7 +68,7 @@ type recommender struct { recommendationPostProcessor []RecommendationPostProcessor } -func (r *recommender) GetClusterState() *model.ClusterState { +func (r *recommender) GetClusterState() model.ClusterState { return r.clusterState } @@ -81,13 +81,13 @@ func (r *recommender) UpdateVPAs() { cnt := metrics_recommender.NewObjectCounter() defer cnt.Observe() - for _, observedVpa := range r.clusterState.ObservedVpas { + for _, observedVpa := range r.clusterState.ObservedVPAs() { key := model.VpaID{ Namespace: observedVpa.Namespace, VpaName: observedVpa.Name, } - vpa, found := r.clusterState.Vpas[key] + vpa, found := r.clusterState.VPAs()[key] if !found { continue } @@ -155,7 +155,7 @@ func (r *recommender) RunOnce() { r.clusterStateFeeder.LoadRealTimeMetrics() timer.ObserveStep("LoadMetrics") - klog.V(3).InfoS("ClusterState is tracking", "pods", len(r.clusterState.Pods), "vpas", len(r.clusterState.Vpas)) + klog.V(3).InfoS("ClusterState is tracking", "pods", len(r.clusterState.Pods()), "vpas", len(r.clusterState.VPAs())) r.UpdateVPAs() timer.ObserveStep("UpdateVPAs") @@ -172,7 +172,7 @@ func (r *recommender) RunOnce() { // RecommenderFactory makes instances of Recommender. type RecommenderFactory struct { - ClusterState *model.ClusterState + ClusterState model.ClusterState ClusterStateFeeder input.ClusterStateFeeder ControllerFetcher controllerfetcher.ControllerFetcher diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go index cd0d360d0f4..2dd078a498a 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram.go @@ -278,6 +278,15 @@ func (h *histogram) LoadFromCheckpoint(checkpoint *vpa_types.HistogramCheckpoint } h.totalWeight += checkpoint.TotalWeight + // In some cases where the weight of the max bucket is close (equal or less) to `MaxCheckpointWeight` times epsilon + // and there are buckets with weights slightly higher or equal to epsilon, saving the histogram to a checkpoint and + // then loading it will cause the weights that are close to epsilon to become smaller than epsilon due to rounding errors + // and differences between the load and save algorithm. If one of those weights is the min weight, this will cause the + // histogram to incorrectly become "empty" and the `Percentile(...)` function to always return 0. + // To cover for such cases, the min and max buckets are updated here, so that those less than epsilon are dropped. + // For more information check https://github.com/kubernetes/autoscaler/issues/7726 + h.updateMinAndMaxBucket() + return nil } diff --git a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go index 09e2fb1706f..3dfb175b21a 100644 --- a/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go +++ b/vertical-pod-autoscaler/pkg/recommender/util/histogram_test.go @@ -265,6 +265,52 @@ func TestHistogramLoadFromCheckpointReturnsErrorOnNilInput(t *testing.T) { assert.Error(t, err) } +func TestHistogramIsNotEmptyAfterSavingAndLoadingCheckpointsWithBoundaryValues(t *testing.T) { + // There is a specific scenario in which the weights of the minimum and maximum histogram buckets, + // when saved to a VPACheckpoint and subsequently loaded, result in diminished weights for the minimum buckets. + + // This issue arises due to rounding errors when converting float weights to integers in the VPACheckpoint. + // For instance, consider the weights: + // `w1` which approximates but is slightly larger than or equal to `epsilon`, + // `w2` which approximates but is slightly smaller than or equal to (`MaxCheckpointWeight` * `epsilon`) - `epsilon`. + + // When these weights are stored in a VPACheckpoint, they are translated to integers: + // `w1` rounds to `1` (`wi1`), + // `w2` rounds to `MaxCheckpointWeight` (`wi2`). + + // Upon loading from the VPACheckpoint, the histogram reconstructs its weights using a calculated ratio, + // aimed at reverting integer weights back to float values. This ratio is derived from: + // (`w1` + `w2`) / (`wi1` + `wi2`) + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler//pkg/recommender/util/histogram.go#L256-L269 + + // Given the maximum potential values for `w1`, `w2`, `wi1` and `wi2` we arrive at: + // (`epsilon` + `MaxCheckpointWeight` * `epsilon` - `epsilon`) / (1 + MaxCheckpointWeight) = epsilon * `MaxCheckpointWeight` / (1 + MaxCheckpointWeight) + + // Consequently, the maximum value for this ratio is less than `epsilon`, implying that when `w1`, + // initially scaled to `1`, is adjusted by this ratio, its recalculated weight falls short of `epsilon`. + // When the `minBucket`'s weight is less than `epsilon`, the `histogram.IsEmpty()` returns true. + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L181-L183 + // Consequently, the `histogram.Percentile(...)` function will always return 0. + // Reference: https://github.com/kubernetes/autoscaler/blob/aa1d413ea3bf319b56c7b2e65ade1a028e149439/vertical-pod-autoscaler/pkg/recommender/util/histogram.go#L159-L162 + // The same behavior can be observed when there are more than two weights. + + // This test ensures that in such cases the histogram does not become empty. + // For more information check https://github.com/kubernetes/autoscaler/issues/7726 + + histogram := NewHistogram(testHistogramOptions) + histogram.AddSample(1, weightEpsilon, anyTime) + histogram.AddSample(2, (float64(MaxCheckpointWeight)*weightEpsilon - weightEpsilon), anyTime) + assert.False(t, histogram.IsEmpty()) + + checkpoint, err := histogram.SaveToChekpoint() + assert.NoError(t, err) + + newHistogram := NewHistogram(testHistogramOptions) + err = newHistogram.LoadFromCheckpoint(checkpoint) + assert.NoError(t, err) + assert.False(t, newHistogram.IsEmpty()) +} + func areUnique(values ...interface{}) bool { dict := make(map[interface{}]bool) for i, v := range values {