Merge branch 'main' into servicemonitor_api_caps

Author: splichy

.gitignore

@@ -14,3 +14,6 @@ go.work.sum
 
 # Project Specific
 *.csv
+
+# Binary output
+karpenter-provider-aws-*

Makefile

@@ -34,6 +34,10 @@ KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' sigs.k8s.io/karpenter)
 # TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" against
 TEST_SUITE ?= "..."
 
+# Filename when building the binary controller only
+GOARCH ?= $(shell go env GOARCH)
+BINARY_FILENAME = karpenter-provider-aws-$(GOARCH)
+
 help: ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
@@ -132,6 +136,9 @@ image: ## Build the Karpenter controller images using ko build
 	$(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s))
 	$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
 
+binary: ## Build the Karpenter controller binary using go build
+	go build $(GOFLAGS) -o $(BINARY_FILENAME) ./cmd/controller/...
+
 apply: verify image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
 	kubectl apply -f ./pkg/apis/crds/
 	helm upgrade --install karpenter charts/karpenter --namespace ${KARPENTER_NAMESPACE} \

charts/karpenter/templates/_helpers.tpl

@@ -75,6 +75,12 @@ Karpenter image to use
 {{- end }}
 {{- end }}
 
+{{/*
+Karpenter controller container name
+*/}}
+{{- define "karpenter.controller.containerName" -}}
+{{- .Values.controller.containerName | default "controller" -}}
+{{- end -}}
 
 {{/* Get PodDisruptionBudget API Version */}}
 {{- define "karpenter.pdb.apiVersion" -}}

charts/karpenter/templates/deployment.yaml

@@ -60,7 +60,7 @@ spec:
       schedulerName: {{ . | quote }}
       {{- end }}
       containers:
-        - name: {{ .Values.controller.containerName | default "controller" }}
+        - name: {{ include "karpenter.controller.containerName" . }}
           securityContext:
             runAsUser: 65532
             runAsGroup: 65532
@@ -102,7 +102,7 @@ spec:
             - name: MEMORY_LIMIT
               valueFrom:
                 resourceFieldRef:
-                  containerName: controller
+                  containerName: {{ include "karpenter.controller.containerName" . }}
                   divisor: "0"
                   resource: limits.memory
             - name: FEATURE_GATES
@@ -183,16 +183,16 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
         {{- end }}
-        {{- with .Values.controller.sidecarContainer }}
-          {{- toYaml . | nindent 8 }}
-        {{- end }}
-        {{- if and (.Values.controller.sidecarContainer) (or .Values.controller.extraVolumeMounts .Values.controller.sidecarVolumeMounts) }}
+        {{- range .Values.controller.sidecarContainer }}
+        - {{- toYaml . | nindent 10 }}
+        {{- if or $.Values.controller.extraVolumeMounts $.Values.controller.sidecarVolumeMounts }}
           volumeMounts:
-          {{- with .Values.controller.extraVolumeMounts }}
+            {{- with $.Values.controller.extraVolumeMounts }}
             {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- with .Values.controller.sidecarVolumeMounts }}
+            {{- end }}
+            {{- with $.Values.controller.sidecarVolumeMounts }}
             {{- toYaml . | nindent 12 }}
+            {{- end }}
           {{- end }}
         {{- end }}
       {{- with .Values.nodeSelector }}

go.mod

@@ -1,6 +1,6 @@
 module github.com/aws/karpenter-provider-aws
 
-go 1.23.5
+go 1.23.6
 
 require (
 	github.com/Pallinder/go-randomdata v1.2.0
@@ -43,7 +43,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 	sigs.k8s.io/controller-runtime v0.20.1
-	sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6
+	sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2
 	sigs.k8s.io/yaml v1.4.0
 )
 

go.sum

@@ -336,8 +336,8 @@ sigs.k8s.io/controller-runtime v0.20.1 h1:JbGMAG/X94NeM3xvjenVUaBjy6Ui4Ogd/J5Ztj
 sigs.k8s.io/controller-runtime v0.20.1/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
-sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6 h1:AWuTX1D+2+q9sZT2IkMHauj3ZivwVzixZftlO7lJ7ZQ=
-sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6/go.mod h1:0PV2k6Ua1Sc04M6NIOfVXLNGyFnvdwDxaIJriic2L5o=
+sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2 h1:E8ZbRdDrRfAaNgLgOl3qkBGMyKOoDTb7grYEwV6+FBQ=
+sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2/go.mod h1:S+qNY3XwugJTu+UvgAdeNUxWuwQP/gS0uefdrV5wFLE=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

pkg/controllers/interruption/controller.go

@@ -38,16 +38,14 @@ import (
 	"sigs.k8s.io/karpenter/pkg/operator/injection"
 
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
-	nodeclaimutils "sigs.k8s.io/karpenter/pkg/utils/nodeclaim"
 	"sigs.k8s.io/karpenter/pkg/utils/pretty"
 
+	"sigs.k8s.io/karpenter/pkg/events"
+
 	"github.com/aws/karpenter-provider-aws/pkg/cache"
 	interruptionevents "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/events"
 	"github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/sqs"
-	"github.com/aws/karpenter-provider-aws/pkg/utils"
-
-	"sigs.k8s.io/karpenter/pkg/events"
 )
 
 type Action string
@@ -104,14 +102,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
 	if len(sqsMessages) == 0 {
 		return reconcile.Result{RequeueAfter: singleton.RequeueImmediately}, nil
 	}
-	nodeClaimInstanceIDMap, err := c.makeNodeClaimInstanceIDMap(ctx)
-	if err != nil {
-		return reconcile.Result{}, fmt.Errorf("making nodeclaim instance id map, %w", err)
-	}
-	nodeInstanceIDMap, err := c.makeNodeInstanceIDMap(ctx)
-	if err != nil {
-		return reconcile.Result{}, fmt.Errorf("making node instance id map, %w", err)
-	}
+
 	errs := make([]error, len(sqsMessages))
 	workqueue.ParallelizeUntil(ctx, 10, len(sqsMessages), func(i int) {
 		msg, e := c.parseMessage(sqsMessages[i])
@@ -121,7 +112,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
 			errs[i] = c.deleteMessage(ctx, sqsMessages[i])
 			return
 		}
-		if e = c.handleMessage(ctx, nodeClaimInstanceIDMap, nodeInstanceIDMap, msg); e != nil {
+		if e = c.handleMessage(ctx, msg); e != nil {
 			errs[i] = fmt.Errorf("handling message, %w", e)
 			return
 		}
@@ -154,23 +145,35 @@ func (c *Controller) parseMessage(raw *sqstypes.Message) (messages.Message, erro
 }
 
 // handleMessage takes an action against every node involved in the message that is owned by a NodePool
-func (c *Controller) handleMessage(ctx context.Context, nodeClaimInstanceIDMap map[string]*karpv1.NodeClaim,
-	nodeInstanceIDMap map[string]*corev1.Node, msg messages.Message) (err error) {
-
+func (c *Controller) handleMessage(ctx context.Context, msg messages.Message) (err error) {
 	ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("messageKind", msg.Kind()))
 	ReceivedMessages.Inc(map[string]string{messageTypeLabel: string(msg.Kind())})
 
 	if msg.Kind() == messages.NoOpKind {
 		return nil
 	}
 	for _, instanceID := range msg.EC2InstanceIDs() {
-		nodeClaim, ok := nodeClaimInstanceIDMap[instanceID]
-		if !ok {
+		nodeClaimList := &karpv1.NodeClaimList{}
+		if e := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"status.instanceID": instanceID}); e != nil {
+			err = multierr.Append(err, e)
 			continue
 		}
-		node := nodeInstanceIDMap[instanceID]
-		if e := c.handleNodeClaim(ctx, msg, nodeClaim, node); e != nil {
-			err = multierr.Append(err, e)
+		if len(nodeClaimList.Items) == 0 {
+			continue
+		}
+		for _, nodeClaim := range nodeClaimList.Items {
+			nodeList := &corev1.NodeList{}
+			if e := c.kubeClient.List(ctx, nodeList, client.MatchingFields{"spec.instanceID": instanceID}); e != nil {
+				err = multierr.Append(err, e)
+				continue
+			}
+			var node *corev1.Node
+			if len(nodeList.Items) > 0 {
+				node = &nodeList.Items[0]
+			}
+			if e := c.handleNodeClaim(ctx, msg, &nodeClaim, node); e != nil {
+				err = multierr.Append(err, e)
+			}
 		}
 	}
 	MessageLatency.Observe(time.Since(msg.StartTime()).Seconds(), nil)
@@ -254,48 +257,6 @@ func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *karpv1.No
 	}
 }
 
-// makeNodeClaimInstanceIDMap builds a map between the instance id that is stored in the
-// NodeClaim .status.providerID and the NodeClaim
-func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string]*karpv1.NodeClaim, error) {
-	m := map[string]*karpv1.NodeClaim{}
-	nodeClaims, err := nodeclaimutils.ListManaged(ctx, c.kubeClient, c.cloudProvider)
-	if err != nil {
-		return nil, err
-	}
-	for _, nc := range nodeClaims {
-		if nc.Status.ProviderID == "" {
-			continue
-		}
-		id, err := utils.ParseInstanceID(nc.Status.ProviderID)
-		if err != nil || id == "" {
-			continue
-		}
-		m[id] = nc
-	}
-	return m, nil
-}
-
-// makeNodeInstanceIDMap builds a map between the instance id that is stored in the
-// node .spec.providerID and the node
-func (c *Controller) makeNodeInstanceIDMap(ctx context.Context) (map[string]*corev1.Node, error) {
-	m := map[string]*corev1.Node{}
-	nodeList := &corev1.NodeList{}
-	if err := c.kubeClient.List(ctx, nodeList); err != nil {
-		return nil, fmt.Errorf("listing nodes, %w", err)
-	}
-	for i := range nodeList.Items {
-		if nodeList.Items[i].Spec.ProviderID == "" {
-			continue
-		}
-		id, err := utils.ParseInstanceID(nodeList.Items[i].Spec.ProviderID)
-		if err != nil || id == "" {
-			continue
-		}
-		m[id] = &nodeList.Items[i]
-	}
-	return m, nil
-}
-
 func actionForMessage(msg messages.Message) Action {
 	switch msg.Kind() {
 	case messages.ScheduledChangeKind, messages.SpotInterruptionKind, messages.InstanceStoppedKind, messages.InstanceTerminatedKind:

pkg/controllers/interruption/suite_test.go

@@ -84,7 +84,7 @@ func TestAPIs(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	ctx = options.ToContext(ctx, test.Options())
-	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
+	env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.NodeInstanceIDFieldIndexer(ctx), test.NodeClaimInstanceIDFieldIndexer(ctx)))
 	awsEnv = test.NewEnvironment(ctx, env)
 	fakeClock = &clock.FakeClock{}
 	unavailableOfferingsCache = awscache.NewUnavailableOfferings()

pkg/controllers/nodeclass/ami.go

@@ -22,16 +22,26 @@ import (
 
 	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
+	"sigs.k8s.io/karpenter/pkg/utils/pretty"
 
 	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
 )
 
 type AMI struct {
 	amiProvider amifamily.Provider
+	cm          *pretty.ChangeMonitor
+}
+
+func NewAMIReconciler(provider amifamily.Provider) *AMI {
+	return &AMI{
+		amiProvider: provider,
+		cm:          pretty.NewChangeMonitor(),
+	}
 }
 
 func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) {
@@ -46,6 +56,12 @@ func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconc
 		// Returning 'ok' in this case means that the nodeclass will remain in an unready state until the component is restarted.
 		return reconcile.Result{RequeueAfter: time.Minute}, nil
 	}
+	if uniqueAMIs := lo.Uniq(lo.Map(amis, func(a amifamily.AMI, _ int) string {
+		return a.AmiID
+	})); a.cm.HasChanged(fmt.Sprintf("amis/%s", nodeClass.Name), uniqueAMIs) {
+		log.FromContext(ctx).WithValues("ids", uniqueAMIs).V(1).Info("discovered amis")
+	}
+
 	nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1.AMI {
 		reqs := lo.Map(ami.Requirements.NodeSelectorRequirements(), func(item karpv1.NodeSelectorRequirementWithMinValues, _ int) corev1.NodeSelectorRequirement {
 			return item.NodeSelectorRequirement

pkg/controllers/nodeclass/controller.go

@@ -75,7 +75,7 @@ func NewController(kubeClient client.Client, recorder events.Recorder, subnetPro
 		kubeClient:             kubeClient,
 		recorder:               recorder,
 		launchTemplateProvider: launchTemplateProvider,
-		ami:                    &AMI{amiProvider: amiProvider},
+		ami:                    NewAMIReconciler(amiProvider),
 		subnet:                 &Subnet{subnetProvider: subnetProvider},
 		securityGroup:          &SecurityGroup{securityGroupProvider: securityGroupProvider},
 		instanceProfile:        &InstanceProfile{instanceProfileProvider: instanceProfileProvider},

pkg/operator/operator.go

@@ -26,12 +26,13 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/middleware"
-	config "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	"github.com/aws/aws-sdk-go-v2/service/eks"
 	"github.com/aws/aws-sdk-go-v2/service/iam"
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/aws/smithy-go"
 	"github.com/patrickmn/go-cache"
@@ -66,6 +67,7 @@ import (
 	ssmp "github.com/aws/karpenter-provider-aws/pkg/providers/ssm"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/subnet"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/version"
+	"github.com/aws/karpenter-provider-aws/pkg/utils"
 )
 
 func init() {
@@ -185,6 +187,10 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		launchTemplateProvider,
 	)
 
+	// Setup field indexers on instanceID -- specifically for the interruption controller
+	if options.FromContext(ctx).InterruptionQueue != "" {
+		SetupIndexers(ctx, operator.Manager)
+	}
 	return ctx, &Operator{
 		Operator:                  operator,
 		Config:                    cfg,
@@ -273,3 +279,26 @@ func KubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n
 	}
 	return kubeDNSIP, nil
 }
+
+func SetupIndexers(ctx context.Context, mgr manager.Manager) {
+	lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &karpv1.NodeClaim{}, "status.instanceID", func(o client.Object) []string {
+		if o.(*karpv1.NodeClaim).Status.ProviderID == "" {
+			return nil
+		}
+		id, e := utils.ParseInstanceID(o.(*karpv1.NodeClaim).Status.ProviderID)
+		if e != nil || id == "" {
+			return nil
+		}
+		return []string{id}
+	}), "failed to setup nodeclaim instanceID indexer")
+	lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &corev1.Node{}, "spec.instanceID", func(o client.Object) []string {
+		if o.(*corev1.Node).Spec.ProviderID == "" {
+			return nil
+		}
+		id, e := utils.ParseInstanceID(o.(*corev1.Node).Spec.ProviderID)
+		if e != nil || id == "" {
+			return nil
+		}
+		return []string{id}
+	}), "failed to setup node instanceID indexer")
+}