Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions pkg/controllers/state/statenodepool.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,9 +193,19 @@ func (n *NodePoolState) UpdateNodeClaim(nodeClaim *v1.NodeClaim, markedForDeleti
// If our node/nodeclaim is marked for deletion, we need to make sure that we delete it
if markedForDeletion {
n.MarkNodeClaimDeleting(npName, nodeClaim.Name)
} else {
n.MarkNodeClaimActive(npName, nodeClaim.Name)
return
}
// While the disruption controller has the NodeClaim marked as disrupting (static NodeClaims only,
// see MarkNodeClaimPendingDisruption), it must stay out of Active. Otherwise, an informer reconcile
// of this NodeClaim (e.g. of the DisruptionReason condition patch itself) would race the disruption
// controller and move it back to Active, inflating the running count and causing the static
// deprovisioner to delete the in-flight replacement NodeClaim. We key off of the condition, rather
// than our own PendingDisruption tracking, so that this self-heals once the disruption controller
// clears the condition on an abandoned/failed disruption command (see ClearNodeClaimsCondition).
if nodeClaim.StatusConditions().Get(v1.ConditionTypeDisruptionReason).IsTrue() {
return
}
n.MarkNodeClaimActive(npName, nodeClaim.Name)
}

func (n *NodePoolState) ensureNodePoolEntry(np string) {
Expand Down
32 changes: 32 additions & 0 deletions pkg/controllers/state/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2727,6 +2727,38 @@ var _ = Describe("NodePoolState Tracking", func() {
Expect(deleting).To(Equal(0))
Expect(pendingdisruption).To(Equal(2))
})

It("should not revert a NodeClaim from PendingDisruption back to Active on an unrelated informer reconcile", func() {
cluster.NodePoolState.MarkNodeClaimPendingDisruption(nodePool.Name, nodeClaim.Name)
running, deleting, pendingdisruption := cluster.NodePoolState.GetNodeCount(nodePool.Name)
Expect(running).To(Equal(0))
Expect(deleting).To(Equal(0))
Expect(pendingdisruption).To(Equal(1))

// Simulate the NodeClaim informer reconciling a status update on the NodeClaim (e.g. the
// Disrupting status condition patch) while it's still PendingDisruption and not yet
// MarkForDeletion'd. This should not move it back into Active.
nodeClaim.StatusConditions().SetTrueWithReason(v1.ConditionTypeDisruptionReason, "Drifted", "Drifted")
ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaim))

running, deleting, pendingdisruption = cluster.NodePoolState.GetNodeCount(nodePool.Name)
Expect(running).To(Equal(0))
Expect(deleting).To(Equal(0))
Expect(pendingdisruption).To(Equal(1))

// Once the disruption controller abandons the command and clears the DisruptionReason
// condition (state.ClearNodeClaimsCondition), the next informer reconcile should recover
// the NodeClaim back to Active rather than leaving it stuck as PendingDisruption forever.
_ = nodeClaim.StatusConditions().Clear(v1.ConditionTypeDisruptionReason)
ExpectApplied(ctx, env.Client, nodeClaim)
ExpectReconcileSucceeded(ctx, nodeClaimController, client.ObjectKeyFromObject(nodeClaim))

running, deleting, pendingdisruption = cluster.NodePoolState.GetNodeCount(nodePool.Name)
Expect(running).To(Equal(1))
Expect(deleting).To(Equal(0))
Expect(pendingdisruption).To(Equal(0))
})
})

Context("DeleteNodeClaim", func() {
Expand Down