Merge pull request #3 from aidenkeating/add-s3-support

add s3 bucket deletion via an action engine

Author: aidenkeating

cmd/cli/cleanup.go

@@ -3,6 +3,9 @@ package main
 import (
 	"fmt"
 	"os"
+	"time"
+
+	"github.com/sirupsen/logrus"
 
 	"github.com/olekukonko/tablewriter"
 
@@ -21,6 +24,7 @@ var cleanupCmd = &cobra.Command{
 	Short: "delete aws resources for an rhmi cluster",
 	Args:  cobra.MinimumNArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
+		//pre-req checks
 		clusterId := args[0]
 		region, err := cmd.Flags().GetString("region")
 		if err != nil {
@@ -34,7 +38,19 @@ var cleanupCmd = &cobra.Command{
 		if err != nil {
 			exitError(fmt.Sprintf("failed to get dry run from flag: %+v", err), exitCodeErrUnknown)
 		}
-
+		watch, err := cmd.Flags().GetBool("watch")
+		if err != nil {
+			exitError(fmt.Sprintf("failed to get watch from flag: %+v", err), exitCodeErrUnknown)
+		}
+		types, err := cmd.Flags().GetStringSlice("types")
+		if err != nil {
+			exitError(fmt.Sprintf("failed to get types from flag: %+v", err), exitCodeErrUnknown)
+		}
+		//ensure the output format is supported
+		if outputFormat != "table" {
+			exitError(fmt.Sprintf("output format %s not supported, use table", outputFormat), exitCodeErrKnown)
+		}
+		//setup aws session
 		awsKeyID := os.Getenv("AWS_ACCESS_KEY_ID")
 		if awsKeyID == "" {
 			exitError("AWS_ACCESS_KEY_ID env var must be defined", exitCodeErrKnown)
@@ -47,19 +63,53 @@ var cleanupCmd = &cobra.Command{
 			Region:      aws.String(region),
 			Credentials: credentials.NewStaticCredentials(awsKeyID, awsSecretKey, ""),
 		}))
-		clusterService := awsclusterservice.NewDefaultClient(awsSession, logger)
-		report, err := clusterService.DeleteResourcesForCluster(clusterId, map[string]string{}, dryRun)
-		if err != nil {
-			exitError(fmt.Sprintf("failed to cleanup resources for cluster, clusterId=%s: %+v", clusterId, err), exitCodeErrUnknown)
+		clusterService := buildAWSClientFromTypes(awsSession, types, logger)
+		//this could probably leverage channels
+		var currentReport *clusterservice.Report
+		for {
+			newReport, err := clusterService.DeleteResourcesForCluster(clusterId, map[string]string{}, dryRun)
+			if err != nil {
+				exitError(fmt.Sprintf("failed to cleanup resources for cluster, clusterId=%s: %+v", clusterId, err), exitCodeErrUnknown)
+			}
+			if currentReport == nil {
+				currentReport = newReport
+			}
+			currentReport.MergeForward(newReport)
+			printReportTable(currentReport)
+			if !watch {
+				break
+			}
+			logger.Debug("watch is enabled, waiting 10 seconds before re-invoking")
+			time.Sleep(time.Second * 10)
 		}
-		// we only support table here...
-		if outputFormat != "table" {
-			exitError(fmt.Sprintf("output format %s not supported, use table", outputFormat), exitCodeErrKnown)
-		}
-		printReportTable(report)
 	},
 }
 
+func buildAWSClientFromTypes(awsSession *session.Session, types []string, logger *logrus.Entry) *awsclusterservice.Client {
+	if types == nil || len(types) == 0 {
+		return awsclusterservice.NewDefaultClient(awsSession, logger)
+	}
+	client := &awsclusterservice.Client{
+		Logger:           logger,
+		ResourceManagers: make([]awsclusterservice.ClusterResourceManager, 0),
+	}
+	for _, t := range types {
+		switch t {
+		case "rds:instance":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultRDSInstanceManager(awsSession, logger))
+		case "rds:snapshot":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultRDSSnapshotManager(awsSession, logger))
+		case "s3":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultS3Engine(awsSession, logger))
+		case "elasticache:replicationgroup":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultElastiCacheEngine(awsSession, logger))
+		default:
+			logger.Debugf("could not find resource manager for specified type %s", t)
+		}
+	}
+	return client
+}
+
 func printReportTable(report *clusterservice.Report) {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"ID", "Name", "Action", "Status"})
@@ -74,4 +124,6 @@ func init() {
 	cleanupCmd.Flags().StringP("output", "o", "table", "set output format")
 	cleanupCmd.Flags().StringP("region", "r", "eu-west-1", "region to delete resources in")
 	cleanupCmd.Flags().BoolP("dry-run", "d", true, "skip performing actions")
+	cleanupCmd.Flags().BoolP("watch", "w", false, "poll actions being performed indefinitely")
+	cleanupCmd.Flags().StringSliceP("types", "t", []string{}, "resource types to cleanup")
 }

hack/aws-ec2-create-subnet.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+REGION="us-east-1"
+VPC_ID=$(aws ec2 describe-vpcs --filters Name=isDefault,Values=true --region "$REGION" | jq -r '.Vpcs[0].VpcId')
+
+echo "creating subnet in vpc $VPC_ID in region $REGION"
+SUBNET_ID=$(aws ec2 create-subnet --region "$REGION" --cidr-block 172.31.128.0/20 --vpc-id "$VPC_ID" | jq -r '.Subnet.SubnetId')
+aws ec2 create-tags --resources "$SUBNET_ID" --tags Key=integreatly.org/clusterID,Value=cluster-service --region us-east-1

hack/aws-elasticache-create-replicationgroup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+REGION="us-east-1"
+RG_ID=${1:-cluster-service-deleteme}
+
+echo "creating replication group in region $REGION"
+aws elasticache create-replication-group --region "$REGION" \
+  --replication-group-id "$RG_ID" --tags "Key=integreatly.org/clusterID,Value=cluster-service" \
+  --replication-group-description "$RG_ID" --engine redis \
+  --cache-node-type cache.t2.micro

hack/aws-rds-create-db.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DB_ID=${1:-cluster-service-deleteme}
+REGION="us-east-1"
+PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+echo "creating test rds instance"
+aws rds create-db-instance --db-instance-identifier "$DB_ID" --db-instance-class db.t2.micro \
+  --engine postgres --allocated-storage=20 --region "$REGION" --master-username postgres \
+  --master-user-password "$PASSWORD" --copy-tags-to-snapshot --tags Key=integreatly.org/clusterID,Value=cluster-service

hack/aws-rds-create-snapshot.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+DB_ID=${1:-cluster-service-deleteme}
+SNAPSHOT_ID=${1:-cluster-service-deleteme}
+REGION="us-east-1"
+SNAPSHOT_SUFFIX=$(cat /dev/urandom | env LC_CTYPE=C tr -dc 'a-zA-Z0-9' | fold -w 4 | head -n 1)
+
+echo "creating test rds instance"
+aws rds create-db-snapshot --db-instance-identifier "$DB_ID" --db-snapshot-identifier "$SNAPSHOT_ID-$SNAPSHOT_SUFFIX" --region "$REGION"

hack/aws-s3-create-bucket.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+BUCKET_ID=${1:-cluster-service-deleteme}
+REGION="us-east-1"
+
+echo "creating test bucket"
+aws s3api create-bucket --bucket="$BUCKET_ID" --region="$REGION"
+echo "tagging bucket with cluster id tag"
+aws s3api put-bucket-tagging --bucket="$BUCKET_ID" --region="$REGION" --tagging 'TagSet=[{Key=integreatly.org/clusterID,Value=cluster-service}]'
+echo "syncing test files from $DIR to bucket"
+aws s3 sync "$DIR/files" "s3://$BUCKET_ID" --region="$REGION"

hack/files/test-file.txt

@@ -0,0 +1 @@
+i am a test file for use with hack scripts

pkg/aws/client.go

@@ -12,28 +12,30 @@ import (
 var _ clusterservice.Client = &Client{}
 
 type Client struct {
-	actionEngines []ActionEngine
-	logger        *logrus.Entry
+	ResourceManagers []ClusterResourceManager
+	Logger           *logrus.Entry
 }
 
 func NewDefaultClient(awsSession *session.Session, logger *logrus.Entry) *Client {
 	log := logger.WithField("cluster_service_provider", "aws")
-	rdsEngine := NewDefaultRDSEngine(awsSession, logger)
+	rdsEngine := NewDefaultRDSInstanceManager(awsSession, logger)
+	rdsSnapshotManager := NewDefaultRDSSnapshotManager(awsSession, logger)
+	s3Engine := NewDefaultS3Engine(awsSession, logger)
 	elasticacheEngine := NewDefaultElastiCacheEngine(awsSession, logger)
 	return &Client{
-		actionEngines: []ActionEngine{rdsEngine, elasticacheEngine},
-		logger:        log,
+		ResourceManagers: []ClusterResourceManager{rdsEngine, elasticacheEngine, s3Engine, rdsSnapshotManager},
+		Logger:           log,
 	}
 }
 
 //DeleteResourcesForCluster Delete AWS resources based on tags using provided action engines
 func (c *Client) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) (*clusterservice.Report, error) {
-	logger := c.logger.WithField("clusterId", clusterId)
+	logger := c.Logger.WithFields(logrus.Fields{loggingKeyClusterID: clusterId, loggingKeyDryRun: dryRun})
 	logger.Debugf("deleting resources for cluster")
 	report := &clusterservice.Report{}
-	for _, engine := range c.actionEngines {
-		engineLogger := logger.WithField("engine", engine.GetName())
-		engineLogger.Debugf("found logger")
+	for _, engine := range c.ResourceManagers {
+		engineLogger := logger.WithField(loggingKeyEngine, engine.GetName())
+		engineLogger.Debugf("found Logger")
 		reportItems, err := engine.DeleteResourcesForCluster(clusterId, tags, dryRun)
 		if err != nil {
 			return nil, errors.WrapLog(err, fmt.Sprintf("failed to run engine %s", engine.GetName()), engineLogger)

pkg/aws/client_test.go

@@ -18,7 +18,7 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 	}
 
 	type fields struct {
-		actionEngines func() []ActionEngine
+		actionEngines func() []ClusterResourceManager
 		logger        *logrus.Entry
 	}
 	type args struct {
@@ -36,7 +36,7 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 		{
 			name: "error when an engine fails",
 			fields: fields{
-				actionEngines: func() []ActionEngine {
+				actionEngines: func() []ClusterResourceManager {
 					fakeEngine, err := fakeActionEngine(func(e *ActionEngineMock) error {
 						e.DeleteResourcesForClusterFunc = func(clusterId string, tags map[string]string, dryRun bool) (items []*clusterservice.ReportItem, e error) {
 							return nil, errors.New("")
@@ -46,7 +46,7 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 					if err != nil {
 						t.Fatal(err)
 					}
-					return []ActionEngine{fakeEngine}
+					return []ClusterResourceManager{fakeEngine}
 				},
 				logger: fakeLogger,
 			},
@@ -60,7 +60,7 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 		{
 			name: "multiple engine report items are appended",
 			fields: fields{
-				actionEngines: func() []ActionEngine {
+				actionEngines: func() []ClusterResourceManager {
 					fakeEngine, err := fakeActionEngine(func(e *ActionEngineMock) error {
 						return nil
 					})
@@ -78,7 +78,7 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 					if err != nil {
 						t.Fatal(err)
 					}
-					return []ActionEngine{fakeEngine, fakeDryRunEngine}
+					return []ClusterResourceManager{fakeEngine, fakeDryRunEngine}
 				},
 				logger: fakeLogger,
 			},
@@ -98,8 +98,8 @@ func TestClient_DeleteResourcesForCluster(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			c := &Client{
-				actionEngines: tt.fields.actionEngines(),
-				logger:        tt.fields.logger,
+				ResourceManagers: tt.fields.actionEngines(),
+				Logger:           tt.fields.logger,
 			}
 			got, err := c.DeleteResourcesForCluster(tt.args.clusterId, tt.args.tags, tt.args.dryRun)
 			if err != nil && err.Error() != tt.wantErr {

pkg/aws/elasticache.go

@@ -1,6 +1,8 @@
 package aws
 
 import (
+	"strings"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/elasticache"
@@ -10,10 +12,9 @@ import (
 	"github.com/integr8ly/cluster-service/pkg/clusterservice"
 	"github.com/integr8ly/cluster-service/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"strings"
 )
 
-var _ ActionEngine = &ElasticacheEngine{}
+var _ ClusterResourceManager = &ElasticacheEngine{}
 
 type ElasticacheEngine struct {
 	elasticacheClient elasticacheiface.ElastiCacheAPI