Try gracefully shutting down before deleting

Signed-off-by: Johan Siebens <[email protected]>

Author: jsiebens

plugin/digitalocean.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"time"
 
 	"github.com/digitalocean/godo"
@@ -27,12 +28,12 @@ type dropletTemplate struct {
 	userData   string
 }
 
-func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *dropletTemplate, config map[string]string) error {
-	log := t.logger.With("action", "scale_out", "tag", template.nodeClass, "count", num)
+func (t *TargetPlugin) scaleOut(ctx context.Context, desired, diff int64, template *dropletTemplate, config map[string]string) error {
+	log := t.logger.With("action", "scale_out", "tag", template.nodeClass, "count", diff)
 
 	log.Debug("creating DigitalOcean droplets")
 
-	for i := int64(0); i < num; i++ {
+	for i := int64(0); i < diff; i++ {
 		createRequest := &godo.DropletCreateRequest{
 			Name:    template.nodeClass + "-" + randstr.String(6),
 			Region:  template.region,
@@ -65,7 +66,7 @@ func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *drople
 
 	log.Debug("successfully created DigitalOcean droplets")
 
-	if err := t.ensureDropletsAreStable(ctx, template); err != nil {
+	if err := t.ensureDropletsAreStable(ctx, template, desired); err != nil {
 		return fmt.Errorf("failed to confirm scale out DigitalOcean droplets: %v", err)
 	}
 
@@ -74,9 +75,9 @@ func (t *TargetPlugin) scaleOut(ctx context.Context, num int64, template *drople
 	return nil
 }
 
-func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *dropletTemplate, config map[string]string) error {
+func (t *TargetPlugin) scaleIn(ctx context.Context, desired, diff int64, template *dropletTemplate, config map[string]string) error {
 
-	ids, err := t.clusterUtils.RunPreScaleInTasks(ctx, config, int(num))
+	ids, err := t.clusterUtils.RunPreScaleInTasks(ctx, config, int(diff))
 	if err != nil {
 		return fmt.Errorf("failed to perform pre-scale Nomad scale in tasks: %v", err)
 	}
@@ -98,9 +99,9 @@ func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *droplet
 		return fmt.Errorf("failed to delete instances: %v", err)
 	}
 
-	log.Debug("successfully deleted DigitalOcean droplets")
+	log.Debug("successfully started deletion process")
 
-	if err := t.ensureDropletsAreStable(ctx, template); err != nil {
+	if err := t.ensureDropletsAreStable(ctx, template, desired); err != nil {
 		return fmt.Errorf("failed to confirm scale in DigitalOcean droplets: %v", err)
 	}
 
@@ -114,11 +115,11 @@ func (t *TargetPlugin) scaleIn(ctx context.Context, num int64, template *droplet
 	return nil
 }
 
-func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dropletTemplate) error {
+func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dropletTemplate, desired int64) error {
 
 	f := func(ctx context.Context) (bool, error) {
-		total, active, err := t.countDroplets(ctx, template)
-		if total == active || err != nil {
+		_, active, err := t.countDroplets(ctx, template)
+		if desired == active || err != nil {
 			return true, err
 		} else {
 			return false, fmt.Errorf("waiting for droplets to become stable")
@@ -130,6 +131,7 @@ func (t *TargetPlugin) ensureDropletsAreStable(ctx context.Context, template *dr
 
 func (t *TargetPlugin) deleteDroplets(ctx context.Context, tag string, instanceIDs map[string]bool) error {
 	// create options. initially, these will be blank
+	var dropletsToDelete []int
 	opt := &godo.ListOptions{}
 	for {
 		droplets, resp, err := t.client.Droplets.ListByTag(ctx, tag, opt)
@@ -140,15 +142,19 @@ func (t *TargetPlugin) deleteDroplets(ctx context.Context, tag string, instanceI
 		for _, d := range droplets {
 			_, ok := instanceIDs[d.Name]
 			if ok {
-				_, err := t.client.Droplets.Delete(ctx, d.ID)
-				if err != nil {
-					return err
-				}
+				go func(dropletId int) {
+					log := t.logger.With("action", "delete", "droplet_id", strconv.Itoa(dropletId))
+					err := shutdownDroplet(dropletId, t.client, log)
+					if err != nil {
+						log.Error("error deleting droplet", err)
+					}
+				}(d.ID)
+				dropletsToDelete = append(dropletsToDelete, d.ID)
 			}
 		}
 
-		// if we are at the last page, break out the for loop
-		if resp.Links == nil || resp.Links.IsLastPage() {
+		// if we deleted all droplets or if we are at the last page, break out the for loop
+		if len(dropletsToDelete) == len(instanceIDs) || resp.Links == nil || resp.Links.IsLastPage() {
 			break
 		}
 

plugin/plugin.go

@@ -123,13 +123,13 @@ func (t *TargetPlugin) Scale(action sdk.ScalingAction, config map[string]string)
 		return fmt.Errorf("failed to describe DigitalOcedroplets: %v", err)
 	}
 
-	num, direction := t.calculateDirection(total, action.Count)
+	diff, direction := t.calculateDirection(total, action.Count)
 
 	switch direction {
 	case "in":
-		err = t.scaleIn(ctx, num, template, config)
+		err = t.scaleIn(ctx, action.Count, diff, template, config)
 	case "out":
-		err = t.scaleOut(ctx, num, template, config)
+		err = t.scaleOut(ctx, action.Count, diff, template, config)
 	default:
 		t.logger.Info("scaling not required", "tag", template.nodeClass,
 			"current_count", total, "strategy_count", action.Count)

plugin/shutdown.go

@@ -0,0 +1,37 @@
+package plugin
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/digitalocean/godo"
+	"github.com/hashicorp/go-hclog"
+)
+
+func shutdownDroplet(
+	dropletId int,
+	client *godo.Client,
+	log hclog.Logger) error {
+
+	// Gracefully power off the droplet.
+	log.Debug("Gracefully shutting down droplet...")
+	_, _, err := client.DropletActions.PowerOff(context.TODO(), dropletId)
+	if err != nil {
+		// If we get an error the first time, actually report it
+		return fmt.Errorf("error shutting down droplet: %s", err)
+	}
+
+	err = waitForDropletState("off", dropletId, client, log, 5*time.Minute)
+	if err != nil {
+		log.Warn("Timeout while waiting to for droplet to become 'off'")
+	}
+
+	log.Debug("Deleting Droplet...")
+	_, err = client.Droplets.Delete(context.TODO(), dropletId)
+	if err != nil {
+		return fmt.Errorf("error deleting droplet: %s", err)
+	}
+
+	return nil
+}

plugin/wait.go

@@ -0,0 +1,60 @@
+package plugin
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/digitalocean/godo"
+	"github.com/hashicorp/go-hclog"
+)
+
+func waitForDropletState(
+	desiredState string, dropletId int,
+	client *godo.Client,
+	log hclog.Logger,
+	timeout time.Duration) error {
+	done := make(chan struct{})
+	defer close(done)
+
+	result := make(chan error, 1)
+	go func() {
+		attempts := 0
+		for {
+			attempts += 1
+
+			log.Debug(fmt.Sprintf("Checking droplet status... (attempt: %d)", attempts))
+			droplet, _, err := client.Droplets.Get(context.TODO(), dropletId)
+			if err != nil {
+				result <- err
+				return
+			}
+
+			if droplet.Status == desiredState {
+				result <- nil
+				return
+			}
+
+			// Wait 3 seconds in between
+			time.Sleep(3 * time.Second)
+
+			// Verify we shouldn't exit
+			select {
+			case <-done:
+				// We finished, so just exit the goroutine
+				return
+			default:
+				// Keep going
+			}
+		}
+	}()
+
+	log.Debug(fmt.Sprintf("Waiting for up to %d seconds for droplet to become %s", timeout/time.Second, desiredState))
+	select {
+	case err := <-result:
+		return err
+	case <-time.After(timeout):
+		err := fmt.Errorf("timeout while waiting to for droplet to become '%s'", desiredState)
+		return err
+	}
+}