feat: on-demand capacity reservation support (#7726)

Author: jmdeal

Makefile

@@ -17,8 +17,9 @@ HELM_OPTS ?= --set serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${K
 			--set controller.resources.requests.memory=1Gi \
 			--set controller.resources.limits.cpu=1 \
 			--set controller.resources.limits.memory=1Gi \
-			--set settings.featureGates.spotToSpotConsolidation=true \
 			--set settings.featureGates.nodeRepair=true \
+			--set settings.featureGates.reservedCapacity=true \
+			--set settings.featureGates.spotToSpotConsolidation=true \
 			--create-namespace
 
 # CR for local builds of Karpenter

charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -239,6 +239,39 @@ spec:
                   x-kubernetes-validations:
                     - message: must have only one blockDeviceMappings with rootVolume
                       rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1
+                capacityReservationSelectorTerms:
+                  description: |-
+                    CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to
+                    determine the set of eligible capacity reservations.
+                  items:
+                    properties:
+                      id:
+                        description: ID is the capacity reservation id in EC2
+                        pattern: ^cr-[0-9a-z]+$
+                        type: string
+                      ownerID:
+                        description: Owner is the owner id for the ami.
+                        pattern: ^[0-9]{12}$
+                        type: string
+                      tags:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          Tags is a map of key/value tags used to select capacity reservations.
+                          Specifying '*' for a value selects all values for a given tag key.
+                        maxProperties: 20
+                        type: object
+                        x-kubernetes-validations:
+                          - message: empty tag keys or values aren't supported
+                            rule: self.all(k, k != '' && self[k] != '')
+                    type: object
+                  maxItems: 30
+                  type: array
+                  x-kubernetes-validations:
+                    - message: expected at least one, got none, ['tags', 'id']
+                      rule: self.all(x, has(x.tags) || has(x.id))
+                    - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term'
+                      rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))'
                 context:
                   description: |-
                     Context is a Reserved field in EC2 APIs
@@ -469,7 +502,7 @@ spec:
                     - message: immutable field changed
                       rule: self == oldSelf
                 securityGroupSelectorTerms:
-                  description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed.
+                  description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed.
                   items:
                     description: |-
                       SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes.
@@ -503,12 +536,12 @@ spec:
                       rule: self.size() != 0
                     - message: expected at least one, got none, ['tags', 'id', 'name']
                       rule: self.all(x, has(x.tags) || has(x.id) || has(x.name))
-                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms'
+                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term'
                       rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))'
-                    - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms'
+                    - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term'
                       rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))'
                 subnetSelectorTerms:
-                  description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed.
+                  description: SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed.
                   items:
                     description: |-
                       SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes.
@@ -537,7 +570,7 @@ spec:
                       rule: self.size() != 0
                     - message: expected at least one, got none, ['tags', 'id']
                       rule: self.all(x, has(x.tags) || has(x.id))
-                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms'
+                    - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term'
                       rule: '!self.all(x, has(x.id) && has(x.tags))'
                 tags:
                   additionalProperties:
@@ -640,6 +673,46 @@ spec:
                       - requirements
                     type: object
                   type: array
+                capacityReservations:
+                  description: |-
+                    CapacityReservations contains the current capacity reservation values that are available to this NodeClass under the
+                    CapacityReservation selectors.
+                  items:
+                    properties:
+                      availabilityZone:
+                        description: The availability zone the capacity reservation is available in.
+                        type: string
+                      endTime:
+                        description: |-
+                          The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter
+                          will no longer be able to launch instances into that reservation.
+                        format: date-time
+                        type: string
+                      id:
+                        description: The id for the capacity reservation.
+                        pattern: ^cr-[0-9a-z]+$
+                        type: string
+                      instanceMatchCriteria:
+                        description: Indicates the type of instance launches the capacity reservation accepts.
+                        enum:
+                          - open
+                          - targeted
+                        type: string
+                      instanceType:
+                        description: The instance type for the capacity reservation.
+                        type: string
+                      ownerID:
+                        description: The ID of the AWS account that owns the capacity reservation.
+                        pattern: ^[0-9]{12}$
+                        type: string
+                    required:
+                      - availabilityZone
+                      - id
+                      - instanceMatchCriteria
+                      - instanceType
+                      - ownerID
+                    type: object
+                  type: array
                 conditions:
                   description: Conditions contains signals for health and readiness
                   items:

charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml

@@ -137,7 +137,7 @@ spec:
                           - message: label "kubernetes.io/hostname" is restricted
                             rule: self != "kubernetes.io/hostname"
                           - message: label domain "karpenter.k8s.aws" is restricted
-                            rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
+                            rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
                       minValues:
                         description: |-
                           This field is ALPHA and can be dropped or replaced at any time

charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml

@@ -210,7 +210,7 @@ spec:
                             - message: label "kubernetes.io/hostname" is restricted
                               rule: self.all(x, x != "kubernetes.io/hostname")
                             - message: label domain "karpenter.k8s.aws" is restricted
-                              rule: self.all(x, x in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws"))
+                              rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws"))
                       type: object
                     spec:
                       description: |-
@@ -283,7 +283,7 @@ spec:
                                   - message: label "kubernetes.io/hostname" is restricted
                                     rule: self != "kubernetes.io/hostname"
                                   - message: label domain "karpenter.k8s.aws" is restricted
-                                    rule: self in ["karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
+                                    rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws")
                               minValues:
                                 description: |-
                                   This field is ALPHA and can be dropped or replaced at any time

charts/karpenter/templates/deployment.yaml

@@ -107,7 +107,7 @@ spec:
                   divisor: "0"
                   resource: limits.memory
             - name: FEATURE_GATES
-              value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }}"
+              value: "ReservedCapacity={{ .Values.settings.featureGates.reservedCapacity }},SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }}"
           {{- with .Values.settings.batchMaxDuration }}
             - name: BATCH_MAX_DURATION
               value: "{{ . }}"

charts/karpenter/values.yaml

@@ -184,9 +184,12 @@ settings:
   # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates
   # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features
   featureGates:
-    # -- spotToSpotConsolidation is ALPHA and is disabled by default.
-    # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation.
-    spotToSpotConsolidation: false
     # -- nodeRepair is ALPHA and is disabled by default.
     # Setting this to true will enable node repair.
     nodeRepair: false
+    # -- reservedCapacity is ALPHA and is disabled by default.
+    # Setting this will enable native on-demand capacity reservation support.
+    reservedCapacity: false
+    # -- spotToSpotConsolidation is ALPHA and is disabled by default.
+    # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation.
+    spotToSpotConsolidation: false

cmd/controller/main.go

@@ -15,6 +15,7 @@ limitations under the License.
 package main
 
 import (
+	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	"github.com/aws/karpenter-provider-aws/pkg/cloudprovider"
 	"github.com/aws/karpenter-provider-aws/pkg/controllers"
 	"github.com/aws/karpenter-provider-aws/pkg/operator"
@@ -23,6 +24,7 @@ import (
 	corecontrollers "sigs.k8s.io/karpenter/pkg/controllers"
 	"sigs.k8s.io/karpenter/pkg/controllers/state"
 	coreoperator "sigs.k8s.io/karpenter/pkg/operator"
+	karpoptions "sigs.k8s.io/karpenter/pkg/operator/options"
 )
 
 func main() {
@@ -35,10 +37,15 @@ func main() {
 		op.GetClient(),
 		op.AMIProvider,
 		op.SecurityGroupProvider,
+		op.CapacityReservationProvider,
 	)
 	cloudProvider := metrics.Decorate(awsCloudProvider)
 	clusterState := state.NewCluster(op.Clock, op.GetClient(), cloudProvider)
 
+	if karpoptions.FromContext(ctx).FeatureGates.ReservedCapacity {
+		v1.CapacityReservationsEnabled = true
+	}
+
 	op.
 		WithControllers(ctx, corecontrollers.NewControllers(
 			ctx,
@@ -69,6 +76,7 @@ func main() {
 			op.LaunchTemplateProvider,
 			op.VersionProvider,
 			op.InstanceTypesProvider,
+			op.CapacityReservationProvider,
 		)...).
 		Start(ctx)
 }