Skip to content

Commit 0ebd3ed

Browse files
DerekFrankDerekFrank
authored
Merge branch 'kubernetes-sigs:main' into main
2 parents b476e5e + af2b998 commit 0ebd3ed

File tree

24 files changed

+676
-35
lines changed

24 files changed

+676
-35
lines changed

‎kwok/charts/crds/karpenter.sh_nodepools.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,12 @@ spec:
498498
- type
499499
type: object
500500
type: array
501+
nodeClassObservedGeneration:
502+
description: |-
503+
NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match
504+
the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset
505+
format: int64
506+
type: integer
501507
resources:
502508
additionalProperties:
503509
anyOf:

‎pkg/apis/crds/karpenter.sh_nodepools.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,12 @@ spec:
496496
- type
497497
type: object
498498
type: array
499+
nodeClassObservedGeneration:
500+
description: |-
501+
NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match
502+
the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset
503+
format: int64
504+
type: integer
499505
resources:
500506
additionalProperties:
501507
anyOf:

‎pkg/apis/v1/nodepool_status.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,19 @@ const (
2727
ConditionTypeValidationSucceeded = "ValidationSucceeded"
2828
// ConditionTypeNodeClassReady = "NodeClassReady" condition indicates that underlying nodeClass was resolved and is reporting as Ready
2929
ConditionTypeNodeClassReady = "NodeClassReady"
30+
// ConditionTypeNodeRegistrationHealthy = "NodeRegistrationHealthy" condition indicates if a misconfiguration exists that is preventing successful node launch/registrations that requires manual investigation
31+
ConditionTypeNodeRegistrationHealthy = "NodeRegistrationHealthy"
3032
)
3133

3234
// NodePoolStatus defines the observed state of NodePool
3335
type NodePoolStatus struct {
3436
// Resources is the list of resources that have been provisioned.
3537
// +optional
3638
Resources v1.ResourceList `json:"resources,omitempty"`
39+
// NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match
40+
// the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset
41+
// +optional
42+
NodeClassObservedGeneration int64 `json:"nodeClassObservedGeneration,omitempty"`
3743
// Conditions contains signals for health and readiness
3844
// +optional
3945
Conditions []status.Condition `json:"conditions,omitempty"`

‎pkg/controllers/controllers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import (
5050
nodepoolcounter "sigs.k8s.io/karpenter/pkg/controllers/nodepool/counter"
5151
nodepoolhash "sigs.k8s.io/karpenter/pkg/controllers/nodepool/hash"
5252
nodepoolreadiness "sigs.k8s.io/karpenter/pkg/controllers/nodepool/readiness"
53+
nodepoolregistrationhealth "sigs.k8s.io/karpenter/pkg/controllers/nodepool/registrationhealth"
5354
nodepoolvalidation "sigs.k8s.io/karpenter/pkg/controllers/nodepool/validation"
5455
"sigs.k8s.io/karpenter/pkg/controllers/provisioning"
5556
"sigs.k8s.io/karpenter/pkg/controllers/state"
@@ -88,6 +89,7 @@ func NewControllers(
8889
metricsnodepool.NewController(kubeClient, cloudProvider),
8990
metricsnode.NewController(cluster),
9091
nodepoolreadiness.NewController(kubeClient, cloudProvider),
92+
nodepoolregistrationhealth.NewController(kubeClient, cloudProvider),
9193
nodepoolcounter.NewController(kubeClient, cloudProvider, cluster),
9294
nodepoolvalidation.NewController(kubeClient, cloudProvider),
9395
podevents.NewController(clock, kubeClient, cloudProvider),

‎pkg/controllers/disruption/controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ func (c *Controller) MarkDisrupted(ctx context.Context, m Method, candidates ...
278278
return client.IgnoreNotFound(err)
279279
}
280280
stored := nodeClaim.DeepCopy()
281-
nodeClaim.StatusConditions().SetTrueWithReason(v1.ConditionTypeDisruptionReason, v1.ConditionTypeDisruptionReason, string(m.Reason()))
281+
nodeClaim.StatusConditions().SetTrueWithReason(v1.ConditionTypeDisruptionReason, string(m.Reason()), string(m.Reason()))
282282
return client.IgnoreNotFound(c.kubeClient.Status().Patch(ctx, nodeClaim, client.MergeFrom(stored)))
283283
})...)
284284
}

‎pkg/controllers/node/termination/terminator/events/events.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,12 @@ import (
2626
"sigs.k8s.io/karpenter/pkg/events"
2727
)
2828

29-
func EvictPod(pod *corev1.Pod, message string) events.Event {
29+
func EvictPod(pod *corev1.Pod, reason string) events.Event {
3030
return events.Event{
3131
InvolvedObject: pod,
3232
Type: corev1.EventTypeNormal,
3333
Reason: events.Evicted,
34-
Message: "Evicted pod: " + message,
34+
Message: "Evicted pod: " + reason,
3535
DedupeValues: []string{pod.Name},
3636
}
3737
}

‎pkg/controllers/node/termination/terminator/eviction.go

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,6 @@ func (q *Queue) Reconcile(ctx context.Context) (reconcile.Result, error) {
175175
// Evict returns true if successful eviction call, and false if there was an eviction-related error
176176
func (q *Queue) Evict(ctx context.Context, key QueueKey) bool {
177177
ctx = log.IntoContext(ctx, log.FromContext(ctx).WithValues("Pod", klog.KRef(key.Namespace, key.Name)))
178-
evictionMessage, err := evictionReason(ctx, key, q.kubeClient)
179-
if err != nil {
180-
// XXX(cmcavoy): this should be unreachable, but we log it if it happens
181-
log.FromContext(ctx).V(1).Error(err, "failed looking up pod eviction reason")
182-
}
183178
if err := q.kubeClient.SubResource("eviction").Create(ctx,
184179
&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: key.Namespace, Name: key.Name}},
185180
&policyv1.Eviction{
@@ -214,18 +209,18 @@ func (q *Queue) Evict(ctx context.Context, key QueueKey) bool {
214209
return false
215210
}
216211
NodesEvictionRequestsTotal.Inc(map[string]string{CodeLabel: "200"})
217-
q.recorder.Publish(terminatorevents.EvictPod(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}}, evictionMessage))
212+
q.recorder.Publish(terminatorevents.EvictPod(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}}, evictionReason(ctx, key, q.kubeClient)))
218213
return true
219214
}
220215

221-
func evictionReason(ctx context.Context, key QueueKey, kubeClient client.Client) (string, error) {
216+
func evictionReason(ctx context.Context, key QueueKey, kubeClient client.Client) string {
222217
nodeClaim, err := node.NodeClaimForNode(ctx, kubeClient, &corev1.Node{Spec: corev1.NodeSpec{ProviderID: key.providerID}})
223218
if err != nil {
224-
return "", err
219+
log.FromContext(ctx).V(1).Error(err, "node has no nodeclaim, failed looking up pod eviction reason")
220+
return ""
225221
}
226-
terminationCondition := nodeClaim.StatusConditions().Get(v1.ConditionTypeDisruptionReason)
227-
if terminationCondition.IsTrue() {
228-
return terminationCondition.Message, nil
222+
if cond := nodeClaim.StatusConditions().Get(v1.ConditionTypeDisruptionReason); cond.IsTrue() {
223+
return cond.Reason
229224
}
230-
return "Forceful Termination", nil
225+
return "Forceful Termination"
231226
}

‎pkg/controllers/nodeclaim/lifecycle/controller.go

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,24 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1.NodeClaim) (re
171171

172172
//nolint:gocyclo
173173
func (c *Controller) finalize(ctx context.Context, nodeClaim *v1.NodeClaim) (reconcile.Result, error) {
174+
// setting the deletion timestamp will bump the generation, so we need to
175+
// perform a no-op for whatever the status condition is currently set to
176+
// so that we bump the observed generation to the latest and prevent the nodeclaim
177+
// root status from entering an `Unknown` state
178+
stored := nodeClaim.DeepCopy()
179+
for _, condition := range nodeClaim.Status.Conditions {
180+
if nodeClaim.StatusConditions().IsDependentCondition(condition.Type) {
181+
nodeClaim.StatusConditions().Set(condition)
182+
}
183+
}
184+
if !equality.Semantic.DeepEqual(stored, nodeClaim) {
185+
if err := c.kubeClient.Status().Patch(ctx, nodeClaim, client.MergeFromWithOptions(stored, client.MergeFromWithOptimisticLock{})); err != nil {
186+
if errors.IsConflict(err) {
187+
return reconcile.Result{Requeue: true}, nil
188+
}
189+
return reconcile.Result{}, err
190+
}
191+
}
174192
if !controllerutil.ContainsFinalizer(nodeClaim, v1.TerminationFinalizer) {
175193
return reconcile.Result{}, nil
176194
}
@@ -226,7 +244,7 @@ func (c *Controller) finalize(ctx context.Context, nodeClaim *v1.NodeClaim) (rec
226244
metrics.NodePoolLabel: nodeClaim.Labels[v1.NodePoolLabelKey],
227245
})
228246
}
229-
stored := nodeClaim.DeepCopy() // The NodeClaim may have been modified in the EnsureTerminated function
247+
stored = nodeClaim.DeepCopy() // The NodeClaim may have been modified in the EnsureTerminated function
230248
controllerutil.RemoveFinalizer(nodeClaim, v1.TerminationFinalizer)
231249
if !equality.Semantic.DeepEqual(stored, nodeClaim) {
232250
// We use client.MergeFromWithOptimisticLock because patching a list with a JSON merge patch

‎pkg/controllers/nodeclaim/lifecycle/liveness.go

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,14 @@ import (
2020
"context"
2121
"time"
2222

23+
"k8s.io/apimachinery/pkg/api/errors"
24+
25+
"k8s.io/apimachinery/pkg/types"
26+
27+
"sigs.k8s.io/controller-runtime/pkg/log"
28+
2329
"k8s.io/utils/clock"
2430
"sigs.k8s.io/controller-runtime/pkg/client"
25-
"sigs.k8s.io/controller-runtime/pkg/log"
2631
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2732

2833
v1 "sigs.k8s.io/karpenter/pkg/apis/v1"
@@ -51,6 +56,12 @@ func (l *Liveness) Reconcile(ctx context.Context, nodeClaim *v1.NodeClaim) (reco
5156
if ttl := registrationTTL - l.clock.Since(registered.LastTransitionTime.Time); ttl > 0 {
5257
return reconcile.Result{RequeueAfter: ttl}, nil
5358
}
59+
if err := l.updateNodePoolRegistrationHealth(ctx, nodeClaim); client.IgnoreNotFound(err) != nil {
60+
if errors.IsConflict(err) {
61+
return reconcile.Result{Requeue: true}, nil
62+
}
63+
return reconcile.Result{}, err
64+
}
5465
// Delete the NodeClaim if we believe the NodeClaim won't register since we haven't seen the node
5566
if err := l.kubeClient.Delete(ctx, nodeClaim); err != nil {
5667
return reconcile.Result{}, client.IgnoreNotFound(err)
@@ -61,6 +72,34 @@ func (l *Liveness) Reconcile(ctx context.Context, nodeClaim *v1.NodeClaim) (reco
6172
metrics.NodePoolLabel: nodeClaim.Labels[v1.NodePoolLabelKey],
6273
metrics.CapacityTypeLabel: nodeClaim.Labels[v1.CapacityTypeLabelKey],
6374
})
64-
6575
return reconcile.Result{}, nil
6676
}
77+
78+
// updateNodePoolRegistrationHealth sets the NodeRegistrationHealthy=False
79+
// on the NodePool if the nodeClaim fails to launch/register
80+
func (l *Liveness) updateNodePoolRegistrationHealth(ctx context.Context, nodeClaim *v1.NodeClaim) error {
81+
nodePoolName := nodeClaim.Labels[v1.NodePoolLabelKey]
82+
if nodePoolName != "" {
83+
nodePool := &v1.NodePool{}
84+
if err := l.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil {
85+
return err
86+
}
87+
if nodePool.StatusConditions().Get(v1.ConditionTypeNodeRegistrationHealthy).IsUnknown() {
88+
stored := nodePool.DeepCopy()
89+
// If the nodeClaim failed to register during the TTL set NodeRegistrationHealthy status condition on
90+
// NodePool to False. If the launch failed get the launch failure reason and message from nodeClaim.
91+
if launchCondition := nodeClaim.StatusConditions().Get(v1.ConditionTypeLaunched); launchCondition.IsTrue() {
92+
nodePool.StatusConditions().SetFalse(v1.ConditionTypeNodeRegistrationHealthy, "RegistrationFailed", "Failed to register node")
93+
} else {
94+
nodePool.StatusConditions().SetFalse(v1.ConditionTypeNodeRegistrationHealthy, launchCondition.Reason, launchCondition.Message)
95+
}
96+
// We use client.MergeFromWithOptimisticLock because patching a list with a JSON merge patch
97+
// can cause races due to the fact that it fully replaces the list on a change
98+
// Here, we are updating the status condition list
99+
if err := l.kubeClient.Status().Patch(ctx, nodePool, client.MergeFromWithOptions(stored, client.MergeFromWithOptimisticLock{})); client.IgnoreNotFound(err) != nil {
100+
return err
101+
}
102+
}
103+
}
104+
return nil
105+
}

‎pkg/controllers/nodeclaim/lifecycle/liveness_test.go

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ package lifecycle_test
1919
import (
2020
"time"
2121

22+
"github.com/awslabs/operatorpkg/status"
23+
24+
operatorpkg "github.com/awslabs/operatorpkg/test/expectations"
2225
. "github.com/onsi/ginkgo/v2"
2326
corev1 "k8s.io/api/core/v1"
2427
"k8s.io/apimachinery/pkg/api/resource"
@@ -78,6 +81,12 @@ var _ = Describe("Liveness", func() {
7881
ExpectFinalizersRemoved(ctx, env.Client, nodeClaim)
7982
if isManagedNodeClaim {
8083
ExpectNotFound(ctx, env.Client, nodeClaim)
84+
operatorpkg.ExpectStatusConditions(ctx, env.Client, 1*time.Minute, nodePool, status.Condition{
85+
Type: v1.ConditionTypeNodeRegistrationHealthy,
86+
Status: metav1.ConditionFalse,
87+
Reason: "RegistrationFailed",
88+
Message: "Failed to register node",
89+
})
8190
} else {
8291
ExpectExists(ctx, env.Client, nodeClaim)
8392
}
@@ -138,6 +147,58 @@ var _ = Describe("Liveness", func() {
138147
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)
139148
nodeClaim = ExpectExists(ctx, env.Client, nodeClaim)
140149

150+
// If the node hasn't registered in the registration timeframe, then we deprovision the nodeClaim
151+
fakeClock.Step(time.Minute * 20)
152+
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)
153+
operatorpkg.ExpectStatusConditions(ctx, env.Client, 1*time.Minute, nodePool, status.Condition{
154+
Type: v1.ConditionTypeNodeRegistrationHealthy,
155+
Status: metav1.ConditionFalse,
156+
Reason: nodeClaim.StatusConditions().Get(v1.ConditionTypeLaunched).Reason,
157+
Message: nodeClaim.StatusConditions().Get(v1.ConditionTypeLaunched).Message,
158+
})
159+
ExpectFinalizersRemoved(ctx, env.Client, nodeClaim)
160+
ExpectNotFound(ctx, env.Client, nodeClaim)
161+
})
162+
It("should not update NodeRegistrationHealthy status condition if it is already set to True", func() {
163+
nodePool.StatusConditions().SetTrue(v1.ConditionTypeNodeRegistrationHealthy)
164+
nodeClaim := test.NodeClaim(v1.NodeClaim{
165+
ObjectMeta: metav1.ObjectMeta{
166+
Labels: map[string]string{
167+
v1.NodePoolLabelKey: nodePool.Name,
168+
},
169+
},
170+
Spec: v1.NodeClaimSpec{
171+
Resources: v1.ResourceRequirements{
172+
Requests: corev1.ResourceList{
173+
corev1.ResourceCPU: resource.MustParse("2"),
174+
corev1.ResourceMemory: resource.MustParse("50Mi"),
175+
corev1.ResourcePods: resource.MustParse("5"),
176+
fake.ResourceGPUVendorA: resource.MustParse("1"),
177+
},
178+
},
179+
},
180+
})
181+
cloudProvider.AllowedCreateCalls = 0 // Don't allow Create() calls to succeed
182+
ExpectApplied(ctx, env.Client, nodePool, nodeClaim)
183+
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)
184+
nodeClaim = ExpectExists(ctx, env.Client, nodeClaim)
185+
186+
// If the node hasn't registered in the registration timeframe, then we deprovision the nodeClaim
187+
fakeClock.Step(time.Minute * 20)
188+
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)
189+
190+
// NodeClaim registration failed, but we should not update the NodeRegistrationHealthy status condition if it is already True
191+
operatorpkg.ExpectStatusConditions(ctx, env.Client, 1*time.Minute, nodePool, status.Condition{Type: v1.ConditionTypeNodeRegistrationHealthy, Status: metav1.ConditionTrue})
192+
ExpectFinalizersRemoved(ctx, env.Client, nodeClaim)
193+
ExpectNotFound(ctx, env.Client, nodeClaim)
194+
})
195+
It("should not block on updating NodeRegistrationHealthy status condition if nodeClaim is not owned by a nodePool", func() {
196+
nodeClaim := test.NodeClaim()
197+
cloudProvider.AllowedCreateCalls = 0 // Don't allow Create() calls to succeed
198+
ExpectApplied(ctx, env.Client, nodeClaim)
199+
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)
200+
nodeClaim = ExpectExists(ctx, env.Client, nodeClaim)
201+
141202
// If the node hasn't registered in the registration timeframe, then we deprovision the nodeClaim
142203
fakeClock.Step(time.Minute * 20)
143204
_ = ExpectObjectReconcileFailed(ctx, env.Client, nodeClaimController, nodeClaim)

‎pkg/controllers/nodeclaim/lifecycle/registration.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ import (
2020
"context"
2121
"fmt"
2222

23+
"k8s.io/apimachinery/pkg/types"
24+
2325
"github.com/samber/lo"
2426
corev1 "k8s.io/api/core/v1"
2527
"k8s.io/apimachinery/pkg/api/equality"
@@ -83,9 +85,37 @@ func (r *Registration) Reconcile(ctx context.Context, nodeClaim *v1.NodeClaim) (
8385
metrics.NodesCreatedTotal.Inc(map[string]string{
8486
metrics.NodePoolLabel: nodeClaim.Labels[v1.NodePoolLabelKey],
8587
})
88+
if err := r.updateNodePoolRegistrationHealth(ctx, nodeClaim); client.IgnoreNotFound(err) != nil {
89+
if errors.IsConflict(err) {
90+
return reconcile.Result{Requeue: true}, nil
91+
}
92+
return reconcile.Result{}, err
93+
}
8694
return reconcile.Result{}, nil
8795
}
8896

97+
// updateNodePoolRegistrationHealth sets the NodeRegistrationHealthy=True
98+
// on the NodePool if the nodeClaim that registered is owned by a NodePool
99+
func (r *Registration) updateNodePoolRegistrationHealth(ctx context.Context, nodeClaim *v1.NodeClaim) error {
100+
nodePoolName := nodeClaim.Labels[v1.NodePoolLabelKey]
101+
if nodePoolName != "" {
102+
nodePool := &v1.NodePool{}
103+
if err := r.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil {
104+
return err
105+
}
106+
stored := nodePool.DeepCopy()
107+
if nodePool.StatusConditions().SetTrue(v1.ConditionTypeNodeRegistrationHealthy) {
108+
// We use client.MergeFromWithOptimisticLock because patching a list with a JSON merge patch
109+
// can cause races due to the fact that it fully replaces the list on a change
110+
// Here, we are updating the status condition list
111+
if err := r.kubeClient.Status().Patch(ctx, nodePool, client.MergeFromWithOptions(stored, client.MergeFromWithOptimisticLock{})); client.IgnoreNotFound(err) != nil {
112+
return err
113+
}
114+
}
115+
}
116+
return nil
117+
}
118+
89119
func (r *Registration) syncNode(ctx context.Context, nodeClaim *v1.NodeClaim, node *corev1.Node) error {
90120
stored := node.DeepCopy()
91121
controllerutil.AddFinalizer(node, v1.TerminationFinalizer)

0 commit comments

Comments
 (0)