Update v0.15.x branch for v0.15.0 release (#1801)

* Detect existence of nodepool stacks and change dependencies accordingly.

* remove used state field - now moved into the stack object

* Add autoscaling:DescribeAutoScalingGroups policy for node drainer. (#1799)

Author: davidmccormick

builtin/files/stack-templates/etcd.json.tmpl

@@ -530,7 +530,7 @@
         }
       },
       "DependsOn": [
-        {{if $.StackExists}}{{if not $.EtcdMigrationEnabled }}{{if $etcdIndex}}"{{$etcdInstance.LogicalNameForIndex (sub $etcdIndex 1)}}",{{end}}{{end}}{{end}}
+        {{if $.Stack.StackExists}}{{if not $.EtcdMigrationEnabled }}{{if $etcdIndex}}"{{$etcdInstance.LogicalNameForIndex (sub $etcdIndex 1)}}",{{end}}{{end}}{{end}}
         {{if $etcdInstance.EIPManaged}}
         "{{$etcdInstance.EIPLogicalName}}",
         {{end}}

builtin/files/stack-templates/node-pool.json.tmpl

@@ -461,6 +461,7 @@
                 {
                   "Action": [
                     "autoscaling:DescribeAutoScalingInstances",
+                    "autoscaling:DescribeAutoScalingGroups",
                     "autoscaling:DescribeLifecycleHooks"
                   ],
                   "Effect": "Allow",

builtin/files/stack-templates/root.json.tmpl

@@ -107,16 +107,18 @@
       },
       "DependsOn": [
         "{{$.ControlPlane.Name}}"
-        {{ if eq $p.NodePoolRollingStrategy "Sequential" -}}
-          {{ if $i -}}
-          ,"{{ (index $.NodePools (sub $i 1)).Name}}"
-          {{ end -}}
-        {{ end -}}
-        {{ if eq $p.NodePoolRollingStrategy "AvailabilityZone" -}}
-          {{- if ne ($.NodePoolAvailabilityZoneDependencies $p $.Subnets) "" }}
-          ,{{ $.NodePoolAvailabilityZoneDependencies $p $.Subnets }}
+        {{- if $p.StackExists }}
+          {{- if eq $p.NodePoolRollingStrategy "Sequential" }}
+            {{- if $i }}
+            ,"{{ (index $.NodePools (sub $i 1)).Name}}"
+            {{- end }}
           {{- end }}
-        {{ end -}}
+          {{- if eq $p.NodePoolRollingStrategy "AvailabilityZone" }}
+            {{- if ne ($.NodePoolAvailabilityZoneDependencies $p $.Subnets) "" }}
+            ,{{ $.NodePoolAvailabilityZoneDependencies $p $.Subnets }}
+            {{- end }}
+          {{- end }}
+        {{- end }}
       ]
     }{{end}}
     {{range $n, $r := .ExtraCfnResources}}

core/root/cluster.go

@@ -228,7 +228,7 @@ func (cl *Cluster) ensureNestedStacksLoaded() error {
 		}
 		npExtras := extras
 		npExtras.Configs = npCfg.Plugins
-		np, err := model.NewWorkerStack(cfg, npCfg, npOpts, npExtras, assetsConfig)
+		np, err := model.NewWorkerStack(cfg, npCfg, npOpts, npExtras, assetsConfig, cl.context())
 		if err != nil {
 			return fmt.Errorf("failed to load node pool #%d: %v", i, err)
 		}

core/root/template_params.go

@@ -177,6 +177,10 @@ func (p nodePool) NeedToExportIAMroles() bool {
 	return p.nodePool.NodePoolConfig.IAMConfig.InstanceProfile.Arn == ""
 }
 
+func (p nodePool) StackExists() bool {
+	return p.nodePool.StackExists
+}
+
 func (p TemplateParams) ControlPlane() controlPlane {
 	return controlPlane{
 		controlPlane: p.cluster.controlPlaneStack,

pkg/api/existing_etcd.go

@@ -2,7 +2,6 @@ package api
 
 // ExistingState describes the existing state of the etcd cluster
 type EtcdExistingState struct {
-	StackExists                    bool
 	EtcdMigrationEnabled           bool
 	EtcdMigrationExistingEndpoints string
 }

pkg/model/context.go

@@ -49,8 +49,10 @@ func (s *Context) stackProvisioner(c *Stack) *cfnstack.Provisioner {
 	)
 }
 
-func (s *Context) InspectEtcdExistingState(c *Config) (api.EtcdExistingState, error) {
+func (s *Context) InspectEtcdExistingState(c *Config) (bool, api.EtcdExistingState, error) {
 	var err error
+	var exists bool
+
 	if s.ProvidedCFInterrogator == nil {
 		s.ProvidedCFInterrogator = cloudformation.New(s.Session)
 	}
@@ -59,18 +61,18 @@ func (s *Context) InspectEtcdExistingState(c *Config) (api.EtcdExistingState, er
 	}
 
 	state := api.EtcdExistingState{}
-	state.StackExists, err = cfnstack.NestedStackExists(s.ProvidedCFInterrogator, c.ClusterName, naming.FromStackToCfnResource(c.Etcd.LogicalName()))
+	exists, err = cfnstack.NestedStackExists(s.ProvidedCFInterrogator, c.ClusterName, naming.FromStackToCfnResource(c.Etcd.LogicalName()))
 	if err != nil {
-		return state, fmt.Errorf("failed to check for existence of etcd cloud-formation stack: %v", err)
+		return exists, state, fmt.Errorf("failed to check for existence of etcd cloud-formation stack: %v", err)
 	}
 	// when the Etcd stack exists we need to check for the MajorMinor version of Etcd running and trigger a migration if different to ours.
-	if state.StackExists {
+	if exists {
 		if state.EtcdMigrationEnabled, err = s.isAMajorEtcdUpgrade(c); err != nil {
-			return state, fmt.Errorf("failed to check existing etcd major minor version: %v", err)
+			return exists, state, fmt.Errorf("failed to check existing etcd major minor version: %v", err)
 		}
 		if state.EtcdMigrationEnabled {
 			if state.EtcdMigrationExistingEndpoints, err = s.lookupExistingEtcdEndpoints(c); err != nil {
-				return state, fmt.Errorf("failed to lookup existing etcd endpoints: %v", err)
+				return exists, state, fmt.Errorf("failed to lookup existing etcd endpoints: %v", err)
 			}
 			logger.Warn("Performing a Major Etcd Version Upgrade: -")
 			logger.Warn("To do this we will spin up new etcd servers and then export the existing kubernetes state to them.")
@@ -80,7 +82,23 @@ func (s *Context) InspectEtcdExistingState(c *Config) (api.EtcdExistingState, er
 			logger.Warn("This operation is best scheduled for a quiet time or in an outage window.")
 		}
 	}
-	return state, nil
+	return exists, state, nil
+}
+
+// Check for the existence of a worker nodepool stack by looking it up in cloudformation.
+func (s *Context) InspectWorkerExistingState(npconf *NodePoolConfig) (bool, error) {
+	var err error
+	var exists bool
+
+	if s.ProvidedCFInterrogator == nil {
+		s.ProvidedCFInterrogator = cloudformation.New(s.Session)
+	}
+
+	exists, err = cfnstack.NestedStackExists(s.ProvidedCFInterrogator, npconf.ClusterName, npconf.NestedStackName())
+	if err != nil {
+		return exists, fmt.Errorf("failed to check worker cloud-formation stack %s: %v", npconf.NestedStackName(), err)
+	}
+	return exists, nil
 }
 
 // isAMajorEtcdUpgrade looks for etcd instances using tag kube-aws:etcd_upgrade_group and the config clusters major-minor version.

pkg/model/stack_new.go

@@ -159,10 +159,11 @@ func NewEtcdStack(conf *Config, opts api.StackTemplateOptions, extras clusterext
 		assetsConfig,
 		func(stack *Stack) (interface{}, error) {
 			// create the context that will be used to build the assets (combination of config + existing state)
-			existingState, err := s.InspectEtcdExistingState(conf)
+			exists, existingState, err := s.InspectEtcdExistingState(conf)
 			if err != nil {
 				return nil, fmt.Errorf("Could not inspect existing etcd state: %v", err)
 			}
+			stack.StackExists = exists
 
 			// Import all the managed subnets from the network stack
 			subnets, err := stack.Config.Subnets.ImportFromNetworkStackRetainingNames()
@@ -222,14 +223,21 @@ func NewEtcdStack(conf *Config, opts api.StackTemplateOptions, extras clusterext
 	)
 }
 
-func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplateOptions, extras clusterextension.ClusterExtension, assetsConfig *credential.CompactAssets) (*Stack, error) {
+func NewWorkerStack(conf *Config, npconf *NodePoolConfig, opts api.StackTemplateOptions, extras clusterextension.ClusterExtension, assetsConfig *credential.CompactAssets, s *Context) (*Stack, error) {
 	logger.Debugf("Generating new Worker stack %s...", npconf.NodePoolName)
 	return newStack(
 		npconf.StackName(),
 		conf,
 		opts,
 		assetsConfig,
 		func(stack *Stack) (interface{}, error) {
+			// create the context that will be used to build the assets (combination of config + existing state)
+			exists, err := s.InspectWorkerExistingState(npconf)
+			if err != nil {
+				return nil, fmt.Errorf("Could not inspect existing nodepool state: %v", err)
+			}
+			stack.StackExists = exists
+
 			return WorkerTmplCtx{
 				Stack:          stack,
 				NodePoolConfig: npconf,

pkg/model/types.go

@@ -19,6 +19,7 @@ type Stack struct {
 	StackName   string
 	S3URI       string
 	ClusterName string
+	StackExists bool
 	Region      api.Region
 
 	Config         *Config