Skip to content

Commit 275ec59

Browse files
authored
Merge branch 'main' into servicemonitor_api_caps
2 parents 91e5d8c + 2cb43bc commit 275ec59

File tree

21 files changed

+158
-110
lines changed

21 files changed

+158
-110
lines changed

.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,6 @@ go.work.sum
1414

1515
# Project Specific
1616
*.csv
17+
18+
# Binary output
19+
karpenter-provider-aws-*

Makefile

+7
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' sigs.k8s.io/karpenter)
3434
# TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" against
3535
TEST_SUITE ?= "..."
3636

37+
# Filename when building the binary controller only
38+
GOARCH ?= $(shell go env GOARCH)
39+
BINARY_FILENAME = karpenter-provider-aws-$(GOARCH)
40+
3741
help: ## Display help
3842
@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
3943

@@ -132,6 +136,9 @@ image: ## Build the Karpenter controller images using ko build
132136
$(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s))
133137
$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
134138

139+
binary: ## Build the Karpenter controller binary using go build
140+
go build $(GOFLAGS) -o $(BINARY_FILENAME) ./cmd/controller/...
141+
135142
apply: verify image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
136143
kubectl apply -f ./pkg/apis/crds/
137144
helm upgrade --install karpenter charts/karpenter --namespace ${KARPENTER_NAMESPACE} \

charts/karpenter/templates/_helpers.tpl

+6
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,12 @@ Karpenter image to use
7575
{{- end }}
7676
{{- end }}
7777

78+
{{/*
79+
Karpenter controller container name
80+
*/}}
81+
{{- define "karpenter.controller.containerName" -}}
82+
{{- .Values.controller.containerName | default "controller" -}}
83+
{{- end -}}
7884

7985
{{/* Get PodDisruptionBudget API Version */}}
8086
{{- define "karpenter.pdb.apiVersion" -}}

charts/karpenter/templates/deployment.yaml

+9-9
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ spec:
6060
schedulerName: {{ . | quote }}
6161
{{- end }}
6262
containers:
63-
- name: {{ .Values.controller.containerName | default "controller" }}
63+
- name: {{ include "karpenter.controller.containerName" . }}
6464
securityContext:
6565
runAsUser: 65532
6666
runAsGroup: 65532
@@ -102,7 +102,7 @@ spec:
102102
- name: MEMORY_LIMIT
103103
valueFrom:
104104
resourceFieldRef:
105-
containerName: controller
105+
containerName: {{ include "karpenter.controller.containerName" . }}
106106
divisor: "0"
107107
resource: limits.memory
108108
- name: FEATURE_GATES
@@ -183,16 +183,16 @@ spec:
183183
{{- toYaml . | nindent 12 }}
184184
{{- end }}
185185
{{- end }}
186-
{{- with .Values.controller.sidecarContainer }}
187-
{{- toYaml . | nindent 8 }}
188-
{{- end }}
189-
{{- if and (.Values.controller.sidecarContainer) (or .Values.controller.extraVolumeMounts .Values.controller.sidecarVolumeMounts) }}
186+
{{- range .Values.controller.sidecarContainer }}
187+
- {{- toYaml . | nindent 10 }}
188+
{{- if or $.Values.controller.extraVolumeMounts $.Values.controller.sidecarVolumeMounts }}
190189
volumeMounts:
191-
{{- with .Values.controller.extraVolumeMounts }}
190+
{{- with $.Values.controller.extraVolumeMounts }}
192191
{{- toYaml . | nindent 12 }}
193-
{{- end }}
194-
{{- with .Values.controller.sidecarVolumeMounts }}
192+
{{- end }}
193+
{{- with $.Values.controller.sidecarVolumeMounts }}
195194
{{- toYaml . | nindent 12 }}
195+
{{- end }}
196196
{{- end }}
197197
{{- end }}
198198
{{- with .Values.nodeSelector }}

go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/aws/karpenter-provider-aws
22

3-
go 1.23.5
3+
go 1.23.6
44

55
require (
66
github.com/Pallinder/go-randomdata v1.2.0
@@ -43,7 +43,7 @@ require (
4343
k8s.io/klog/v2 v2.130.1
4444
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
4545
sigs.k8s.io/controller-runtime v0.20.1
46-
sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6
46+
sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2
4747
sigs.k8s.io/yaml v1.4.0
4848
)
4949

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,8 @@ sigs.k8s.io/controller-runtime v0.20.1 h1:JbGMAG/X94NeM3xvjenVUaBjy6Ui4Ogd/J5Ztj
336336
sigs.k8s.io/controller-runtime v0.20.1/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU=
337337
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
338338
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
339-
sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6 h1:AWuTX1D+2+q9sZT2IkMHauj3ZivwVzixZftlO7lJ7ZQ=
340-
sigs.k8s.io/karpenter v1.2.1-0.20250128194523-2a09110a1cb6/go.mod h1:0PV2k6Ua1Sc04M6NIOfVXLNGyFnvdwDxaIJriic2L5o=
339+
sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2 h1:E8ZbRdDrRfAaNgLgOl3qkBGMyKOoDTb7grYEwV6+FBQ=
340+
sigs.k8s.io/karpenter v1.2.1-0.20250208015555-8e8b99d6bfa2/go.mod h1:S+qNY3XwugJTu+UvgAdeNUxWuwQP/gS0uefdrV5wFLE=
341341
sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
342342
sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
343343
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=

pkg/controllers/interruption/controller.go

+24-63
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,14 @@ import (
3838
"sigs.k8s.io/karpenter/pkg/operator/injection"
3939

4040
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
41-
nodeclaimutils "sigs.k8s.io/karpenter/pkg/utils/nodeclaim"
4241
"sigs.k8s.io/karpenter/pkg/utils/pretty"
4342

43+
"sigs.k8s.io/karpenter/pkg/events"
44+
4445
"github.com/aws/karpenter-provider-aws/pkg/cache"
4546
interruptionevents "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/events"
4647
"github.com/aws/karpenter-provider-aws/pkg/controllers/interruption/messages"
4748
"github.com/aws/karpenter-provider-aws/pkg/providers/sqs"
48-
"github.com/aws/karpenter-provider-aws/pkg/utils"
49-
50-
"sigs.k8s.io/karpenter/pkg/events"
5149
)
5250

5351
type Action string
@@ -104,14 +102,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
104102
if len(sqsMessages) == 0 {
105103
return reconcile.Result{RequeueAfter: singleton.RequeueImmediately}, nil
106104
}
107-
nodeClaimInstanceIDMap, err := c.makeNodeClaimInstanceIDMap(ctx)
108-
if err != nil {
109-
return reconcile.Result{}, fmt.Errorf("making nodeclaim instance id map, %w", err)
110-
}
111-
nodeInstanceIDMap, err := c.makeNodeInstanceIDMap(ctx)
112-
if err != nil {
113-
return reconcile.Result{}, fmt.Errorf("making node instance id map, %w", err)
114-
}
105+
115106
errs := make([]error, len(sqsMessages))
116107
workqueue.ParallelizeUntil(ctx, 10, len(sqsMessages), func(i int) {
117108
msg, e := c.parseMessage(sqsMessages[i])
@@ -121,7 +112,7 @@ func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) {
121112
errs[i] = c.deleteMessage(ctx, sqsMessages[i])
122113
return
123114
}
124-
if e = c.handleMessage(ctx, nodeClaimInstanceIDMap, nodeInstanceIDMap, msg); e != nil {
115+
if e = c.handleMessage(ctx, msg); e != nil {
125116
errs[i] = fmt.Errorf("handling message, %w", e)
126117
return
127118
}
@@ -154,23 +145,35 @@ func (c *Controller) parseMessage(raw *sqstypes.Message) (messages.Message, erro
154145
}
155146

156147
// handleMessage takes an action against every node involved in the message that is owned by a NodePool
157-
func (c *Controller) handleMessage(ctx context.Context, nodeClaimInstanceIDMap map[string]*karpv1.NodeClaim,
158-
nodeInstanceIDMap map[string]*corev1.Node, msg messages.Message) (err error) {
159-
148+
func (c *Controller) handleMessage(ctx context.Context, msg messages.Message) (err error) {
160149
ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("messageKind", msg.Kind()))
161150
ReceivedMessages.Inc(map[string]string{messageTypeLabel: string(msg.Kind())})
162151

163152
if msg.Kind() == messages.NoOpKind {
164153
return nil
165154
}
166155
for _, instanceID := range msg.EC2InstanceIDs() {
167-
nodeClaim, ok := nodeClaimInstanceIDMap[instanceID]
168-
if !ok {
156+
nodeClaimList := &karpv1.NodeClaimList{}
157+
if e := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"status.instanceID": instanceID}); e != nil {
158+
err = multierr.Append(err, e)
169159
continue
170160
}
171-
node := nodeInstanceIDMap[instanceID]
172-
if e := c.handleNodeClaim(ctx, msg, nodeClaim, node); e != nil {
173-
err = multierr.Append(err, e)
161+
if len(nodeClaimList.Items) == 0 {
162+
continue
163+
}
164+
for _, nodeClaim := range nodeClaimList.Items {
165+
nodeList := &corev1.NodeList{}
166+
if e := c.kubeClient.List(ctx, nodeList, client.MatchingFields{"spec.instanceID": instanceID}); e != nil {
167+
err = multierr.Append(err, e)
168+
continue
169+
}
170+
var node *corev1.Node
171+
if len(nodeList.Items) > 0 {
172+
node = &nodeList.Items[0]
173+
}
174+
if e := c.handleNodeClaim(ctx, msg, &nodeClaim, node); e != nil {
175+
err = multierr.Append(err, e)
176+
}
174177
}
175178
}
176179
MessageLatency.Observe(time.Since(msg.StartTime()).Seconds(), nil)
@@ -254,48 +257,6 @@ func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *karpv1.No
254257
}
255258
}
256259

257-
// makeNodeClaimInstanceIDMap builds a map between the instance id that is stored in the
258-
// NodeClaim .status.providerID and the NodeClaim
259-
func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string]*karpv1.NodeClaim, error) {
260-
m := map[string]*karpv1.NodeClaim{}
261-
nodeClaims, err := nodeclaimutils.ListManaged(ctx, c.kubeClient, c.cloudProvider)
262-
if err != nil {
263-
return nil, err
264-
}
265-
for _, nc := range nodeClaims {
266-
if nc.Status.ProviderID == "" {
267-
continue
268-
}
269-
id, err := utils.ParseInstanceID(nc.Status.ProviderID)
270-
if err != nil || id == "" {
271-
continue
272-
}
273-
m[id] = nc
274-
}
275-
return m, nil
276-
}
277-
278-
// makeNodeInstanceIDMap builds a map between the instance id that is stored in the
279-
// node .spec.providerID and the node
280-
func (c *Controller) makeNodeInstanceIDMap(ctx context.Context) (map[string]*corev1.Node, error) {
281-
m := map[string]*corev1.Node{}
282-
nodeList := &corev1.NodeList{}
283-
if err := c.kubeClient.List(ctx, nodeList); err != nil {
284-
return nil, fmt.Errorf("listing nodes, %w", err)
285-
}
286-
for i := range nodeList.Items {
287-
if nodeList.Items[i].Spec.ProviderID == "" {
288-
continue
289-
}
290-
id, err := utils.ParseInstanceID(nodeList.Items[i].Spec.ProviderID)
291-
if err != nil || id == "" {
292-
continue
293-
}
294-
m[id] = &nodeList.Items[i]
295-
}
296-
return m, nil
297-
}
298-
299260
func actionForMessage(msg messages.Message) Action {
300261
switch msg.Kind() {
301262
case messages.ScheduledChangeKind, messages.SpotInterruptionKind, messages.InstanceStoppedKind, messages.InstanceTerminatedKind:

pkg/controllers/interruption/suite_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func TestAPIs(t *testing.T) {
8484

8585
var _ = BeforeSuite(func() {
8686
ctx = options.ToContext(ctx, test.Options())
87-
env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...))
87+
env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.NodeInstanceIDFieldIndexer(ctx), test.NodeClaimInstanceIDFieldIndexer(ctx)))
8888
awsEnv = test.NewEnvironment(ctx, env)
8989
fakeClock = &clock.FakeClock{}
9090
unavailableOfferingsCache = awscache.NewUnavailableOfferings()

pkg/controllers/nodeclass/ami.go

+16
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,26 @@ import (
2222

2323
"github.com/samber/lo"
2424
corev1 "k8s.io/api/core/v1"
25+
"sigs.k8s.io/controller-runtime/pkg/log"
2526
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2627

2728
karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
29+
"sigs.k8s.io/karpenter/pkg/utils/pretty"
2830

2931
v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
3032
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
3133
)
3234

3335
type AMI struct {
3436
amiProvider amifamily.Provider
37+
cm *pretty.ChangeMonitor
38+
}
39+
40+
func NewAMIReconciler(provider amifamily.Provider) *AMI {
41+
return &AMI{
42+
amiProvider: provider,
43+
cm: pretty.NewChangeMonitor(),
44+
}
3545
}
3646

3747
func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) {
@@ -46,6 +56,12 @@ func (a *AMI) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconc
4656
// Returning 'ok' in this case means that the nodeclass will remain in an unready state until the component is restarted.
4757
return reconcile.Result{RequeueAfter: time.Minute}, nil
4858
}
59+
if uniqueAMIs := lo.Uniq(lo.Map(amis, func(a amifamily.AMI, _ int) string {
60+
return a.AmiID
61+
})); a.cm.HasChanged(fmt.Sprintf("amis/%s", nodeClass.Name), uniqueAMIs) {
62+
log.FromContext(ctx).WithValues("ids", uniqueAMIs).V(1).Info("discovered amis")
63+
}
64+
4965
nodeClass.Status.AMIs = lo.Map(amis, func(ami amifamily.AMI, _ int) v1.AMI {
5066
reqs := lo.Map(ami.Requirements.NodeSelectorRequirements(), func(item karpv1.NodeSelectorRequirementWithMinValues, _ int) corev1.NodeSelectorRequirement {
5167
return item.NodeSelectorRequirement

pkg/controllers/nodeclass/controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ func NewController(kubeClient client.Client, recorder events.Recorder, subnetPro
7575
kubeClient: kubeClient,
7676
recorder: recorder,
7777
launchTemplateProvider: launchTemplateProvider,
78-
ami: &AMI{amiProvider: amiProvider},
78+
ami: NewAMIReconciler(amiProvider),
7979
subnet: &Subnet{subnetProvider: subnetProvider},
8080
securityGroup: &SecurityGroup{securityGroupProvider: securityGroupProvider},
8181
instanceProfile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider},

pkg/operator/operator.go

+30-1
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,13 @@ import (
2626

2727
"github.com/aws/aws-sdk-go-v2/aws"
2828
"github.com/aws/aws-sdk-go-v2/aws/middleware"
29-
config "github.com/aws/aws-sdk-go-v2/config"
29+
"github.com/aws/aws-sdk-go-v2/config"
3030
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
3131
"github.com/aws/aws-sdk-go-v2/service/ec2"
3232
"github.com/aws/aws-sdk-go-v2/service/eks"
3333
"github.com/aws/aws-sdk-go-v2/service/iam"
3434
"github.com/aws/aws-sdk-go-v2/service/ssm"
35+
"sigs.k8s.io/controller-runtime/pkg/manager"
3536

3637
"github.com/aws/smithy-go"
3738
"github.com/patrickmn/go-cache"
@@ -66,6 +67,7 @@ import (
6667
ssmp "github.com/aws/karpenter-provider-aws/pkg/providers/ssm"
6768
"github.com/aws/karpenter-provider-aws/pkg/providers/subnet"
6869
"github.com/aws/karpenter-provider-aws/pkg/providers/version"
70+
"github.com/aws/karpenter-provider-aws/pkg/utils"
6971
)
7072

7173
func init() {
@@ -185,6 +187,10 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
185187
launchTemplateProvider,
186188
)
187189

190+
// Setup field indexers on instanceID -- specifically for the interruption controller
191+
if options.FromContext(ctx).InterruptionQueue != "" {
192+
SetupIndexers(ctx, operator.Manager)
193+
}
188194
return ctx, &Operator{
189195
Operator: operator,
190196
Config: cfg,
@@ -273,3 +279,26 @@ func KubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n
273279
}
274280
return kubeDNSIP, nil
275281
}
282+
283+
func SetupIndexers(ctx context.Context, mgr manager.Manager) {
284+
lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &karpv1.NodeClaim{}, "status.instanceID", func(o client.Object) []string {
285+
if o.(*karpv1.NodeClaim).Status.ProviderID == "" {
286+
return nil
287+
}
288+
id, e := utils.ParseInstanceID(o.(*karpv1.NodeClaim).Status.ProviderID)
289+
if e != nil || id == "" {
290+
return nil
291+
}
292+
return []string{id}
293+
}), "failed to setup nodeclaim instanceID indexer")
294+
lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &corev1.Node{}, "spec.instanceID", func(o client.Object) []string {
295+
if o.(*corev1.Node).Spec.ProviderID == "" {
296+
return nil
297+
}
298+
id, e := utils.ParseInstanceID(o.(*corev1.Node).Spec.ProviderID)
299+
if e != nil || id == "" {
300+
return nil
301+
}
302+
return []string{id}
303+
}), "failed to setup node instanceID indexer")
304+
}

0 commit comments

Comments
 (0)