CLM could wait forever for node decommissioning

[AlexanderYastrebov]

CLM could wait forever for node decommissioning (if there is some kind of problem)

cluster-lifecycle-manager/pkg/updatestrategy/clc_update.go

Lines 48 to 89 in 769565c

	func (c *CLCUpdateStrategy) PrepareForRemoval(ctx context.Context, nodePoolDesc *api.NodePool) error {
	c.logger.Infof("Preparing for removal of node pool '%s'", nodePoolDesc.Name)
	for {
	nodePool, err := c.nodePoolManager.GetPool(ctx, nodePoolDesc)
	if err != nil {
	return err
	}
	err = c.nodePoolManager.MarkPoolForDecommission(nodePoolDesc)
	if err != nil {
	return err
	}
	for _, node := range nodePool.Nodes {
	err := c.nodePoolManager.DisableReplacementNodeProvisioning(ctx, node)
	if err != nil {
	return err
	}
	}
	nodes, err := c.markNodes(ctx, nodePool, func(_ *Node) bool {
	return true
	})
	if err != nil {
	return err
	}
	if nodes == 0 {
	return nil
	}
	c.logger.WithField("node-pool", nodePoolDesc.Name).Infof("Waiting for decommissioning of the nodes (%d left)", nodes)
	// wait for CLC to finish removing the nodes
	select {
	case <-ctx.Done():
	return ctx.Err()
	case <-time.After(c.pollingInterval):
	}
	}
	}

cluster-lifecycle-manager/pkg/updatestrategy/clc_update.go

Lines 109 to 140 in 769565c

	func (c *CLCUpdateStrategy) doUpdate(ctx context.Context, nodePoolDesc *api.NodePool) error {
	for {
	if err := ctx.Err(); err != nil {
	return err
	}
	nodePool, err := c.nodePoolManager.GetPool(ctx, nodePoolDesc)
	if err != nil {
	return err
	}
	oldNodes, err := c.markNodes(ctx, nodePool, func(node *Node) bool {
	return node.Generation != nodePool.Generation
	})
	if err != nil {
	return err
	}
	if oldNodes == 0 {
	return nil
	}
	c.logger.WithField("node-pool", nodePoolDesc.Name).Infof("Waiting for decommissioning of old nodes (%d left)", oldNodes)
	// wait for CLC to finish rolling the nodes
	select {
	case <-ctx.Done():
	return ctx.Err()
	case <-time.After(c.pollingInterval):
	}
	}
	}

and therefore does not update other node pools.

See also #154

[AlexanderYastrebov]

Some ideas

• use child context.WithTimeout in the PrepareForRemoval and doUpdate
• do cluster config update before nodepool updates

  cluster-lifecycle-manager/provisioner/clusterpy.go

  Line 424 in 769565c

  return p.apply(ctx, logger, cluster, deletions, manifests)

• do not return early on pool update error

  cluster-lifecycle-manager/provisioner/clusterpy.go

  Lines 400 to 409 in 769565c

  	for _, nodePool := range g.NodePools {
  	err := updater.Update(ctx, nodePool)
  	if err != nil {
  	return err
  	}
  	if err = ctx.Err(); err != nil {
  	return err
  	}
  	}

  cluster-lifecycle-manager/provisioner/node_pools.go

  Lines 206 to 219 in 769565c

  	for _, stack := range orphaned {
  	nodePool := nodePoolStackToNodePool(stack)
  	err := updater.PrepareForRemoval(ctx, nodePool)
  	if err != nil {
  	return err
  	}
  	// delete node pool stack
  	err = p.awsAdapter.DeleteStack(ctx, stack)
  	if err != nil {
  	return err
  	}
  	}