Two fixes to 0.9.9 rc.3 (#1043)

* Fix inability to re-render credentials

Fixes #1042

* The best possible work-around to stabilize apiserver+controller-manager w/ metrics-server

Fixes #1039

Author: mumoshu

core/controlplane/config/encrypted_assets.go

@@ -440,11 +440,6 @@ func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredO
 }
 
 func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool) error {
-	workerCAKeyDefaultSymlinkTo := ""
-	if includeCAKey {
-		workerCAKeyDefaultSymlinkTo = "ca-key.pem"
-	}
-
 	assets := []struct {
 		name             string
 		data             []byte
@@ -453,7 +448,6 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool) error
 	}{
 		{"ca.pem", r.CACert, true, ""},
 		{"worker-ca.pem", r.WorkerCACert, true, "ca.pem"},
-		{"worker-ca-key.pem", r.WorkerCAKey, true, workerCAKeyDefaultSymlinkTo},
 		{"apiserver.pem", r.APIServerCert, true, ""},
 		{"apiserver-key.pem", r.APIServerKey, true, ""},
 		{"worker.pem", r.WorkerCert, true, ""},
@@ -480,6 +474,13 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool) error
 			overwrite        bool
 			ifEmptySymlinkTo string
 		}{"ca-key.pem", r.CAKey, true, ""})
+
+		assets = append(assets, struct {
+			name             string
+			data             []byte
+			overwrite        bool
+			ifEmptySymlinkTo string
+		}{"worker-ca-key.pem", r.WorkerCAKey, true, "ca-key.pem"})
 	}
 
 	for _, asset := range assets {

core/controlplane/config/templates/cloud-config-controller

@@ -781,108 +781,111 @@ write_files:
       #!/bin/bash -e
 
       kubectl() {
-          /usr/bin/docker run --rm --net=host -v /srv/kubernetes:/srv/kubernetes {{.HyperkubeImage.RepoWithTag}} /hyperkube kubectl "$@"
+          # --request-timeout=1s is intended to instruct kubectl to give up discovering unresponsive apiservice(s) in certain periods
+          # so that temporal freakiness/unresponsity of specific apiservice until apiserver/controller-manager fully starts doesn't
+          # affect the whole controller bootstrap process.
+          /usr/bin/docker run --rm --net=host -v /srv/kubernetes:/srv/kubernetes {{.HyperkubeImage.RepoWithTag}} /hyperkube kubectl --request-timeout=1s "$@"
+      }
+
+      ks() {
+        kubectl --namespace kube-system "$@"
+      }
+
+      # Try to batch as many files as possible to reduce the total amount of delay due to wilderness in the API aggregation
+      # See https://github.com/kubernetes-incubator/kube-aws/issues/1039
+      applyall() {
+        kubectl apply -f $(echo "$@" | tr ' ' ',')
       }
 
       while ! kubectl get ns kube-system; do
         echo Waiting until kube-system created.
         sleep 3
       done
 
+      # See https://github.com/kubernetes-incubator/kube-aws/issues/1039#issuecomment-348978375
+      if ks get apiservice v1beta1.metrics.k8s.io && ! ps ax | grep '[h]yperkube proxy'; then
+        echo "apiserver is up but kube-proxy isn't up. We have likely encountered #1039."
+        echo "Temporary deleting the v1beta1.metrics.k8s.io apiservice as a work-around for #1039"
+        ks delete apiservice v1beta1.metrics.k8s.io
+
+        echo Waiting until controller-manager stabilizes and it creates a kube-proxy pod.
+        until ps ax | grep '[h]yperkube proxy'; do
+            echo Sleeping 3 seconds.
+            sleep 3
+        done
+        echo kube-proxy stared. apiserver should be responsive again.
+      fi
+
       mfdir=/srv/kubernetes/manifests
 
       {{ if .UseCalico }}
       /bin/bash /opt/bin/populate-tls-calico-etcd
-      kubectl apply -f "${mfdir}/calico.yaml"
+      applyall "${mfdir}/calico.yaml"
       {{ end }}
 
       {{ if .Experimental.NodeDrainer.Enabled }}
-      for manifest in {kube-node-drainer-ds,kube-node-drainer-asg-status-updater-de}; do
-          kubectl apply -f "${mfdir}/$manifest.yaml"
-      done
+      applyall "${mfdir}/{kube-node-drainer-ds,kube-node-drainer-asg-status-updater-de}".yaml"
       {{ end }}
 
       # Secrets
-      kubectl apply -f "${mfdir}/kubernetes-dashboard-se.yaml"
+      applyall "${mfdir}/kubernetes-dashboard-se.yaml"
 
       # Configmaps
-      for manifest in {kube-dns,kube-proxy}; do
-        kubectl apply -f "${mfdir}/$manifest-cm.yaml"
-      done
+      applyall "${mfdir}"/{kube-dns,kube-proxy}"-cm.yaml"
 
       # Service Accounts
-      for manifest in {kube-dns,heapster,kube-proxy,kubernetes-dashboard,metrics-server}; do
-          kubectl apply -f "${mfdir}/$manifest-sa.yaml"
-      done
+      applyall "${mfdir}"/{kube-dns,heapster,kube-proxy,kubernetes-dashboard,metrics-server}"-sa.yaml"
 
       # Install tiller by default
-      kubectl apply -f "${mfdir}/tiller.yaml"
+      applyall "${mfdir}/tiller.yaml"
 
 {{ if .KubeDns.NodeLocalResolver }}
       # DNS Masq Fix
-      kubectl apply -f "${mfdir}/dnsmasq-node-ds.yaml"
+      applyall "${mfdir}/dnsmasq-node-ds.yaml"
 {{ end }}
 
       # Deployments
-      for manifest in {kube-dns,kube-dns-autoscaler,kubernetes-dashboard,{{ if .Addons.ClusterAutoscaler.Enabled }}cluster-autoscaler,{{ end }}heapster{{ if .KubeResourcesAutosave.Enabled }},kube-resources-autosave{{ end }},metrics-server}; do
-          kubectl apply -f "${mfdir}/$manifest-de.yaml"
-      done
+      applyall "${mfdir}"/{kube-dns,kube-dns-autoscaler,kubernetes-dashboard,{{ if .Addons.ClusterAutoscaler.Enabled }}cluster-autoscaler,{{ end }}heapster{{ if .KubeResourcesAutosave.Enabled }},kube-resources-autosave{{ end }},metrics-server}"-de.yaml"
 
       # Daemonsets
-      for manifest in {kube-proxy,}; do
-          kubectl apply -f "${mfdir}/$manifest-ds.yaml"
-      done
+      applyall "${mfdir}"/kube-proxy"-ds.yaml"
 
       # Services
-      for manifest in {kube-dns,heapster,kubernetes-dashboard,metrics-server}; do
-          kubectl apply -f "${mfdir}/$manifest-svc.yaml"
-      done
+      applyall "${mfdir}"/{kube-dns,heapster,kubernetes-dashboard,metrics-server}"-svc.yaml"
 
       {{- if .Addons.Rescheduler.Enabled }}
-      kubectl apply -f "${mfdir}/kube-rescheduler-de.yaml"
+      applyall "${mfdir}/kube-rescheduler-de.yaml"
       {{- end }}
 
       # API Services
-      for manifest in {metrics-server,}; do
-          kubectl apply -f "${mfdir}/$manifest-apisvc.yaml"
-      done
+      applyall "${mfdir}/metrics-server-apisvc.yaml"
 
       mfdir=/srv/kubernetes/rbac
 
       # Cluster roles and bindings
-      for manifest in {node-extensions,metrics-server}; do
-          kubectl apply -f "${mfdir}/cluster-roles/$manifest.yaml"
-      done
-      for manifest in {kube-admin,system-worker,node,node-proxier,node-extensions,heapster,metrics-server}; do
-          kubectl apply -f "${mfdir}/cluster-role-bindings/$manifest.yaml"
-      done
+      applyall "${mfdir}/cluster-roles"/{node-extensions,metrics-server}".yaml"
+
+      applyall "${mfdir}/cluster-role-bindings"/{kube-admin,system-worker,node,node-proxier,node-extensions,heapster,metrics-server}".yaml"
 
       {{ if .KubernetesDashboard.AdminPrivileges }}
-      kubectl apply -f "${mfdir}/cluster-role-bindings/kubernetes-dashboard-admin.yaml"
+      applyall "${mfdir}/cluster-role-bindings/kubernetes-dashboard-admin.yaml"
       {{- end }}
 
       # Roles and bindings
-      for manifest in {pod-nanny,kubernetes-dashboard}; do
-          kubectl apply -f "${mfdir}/roles/$manifest.yaml"
-      done
-      for manifest in {heapster-nanny,kubernetes-dashboard,metrics-server}; do
-          kubectl apply -f "${mfdir}/role-bindings/$manifest.yaml"
-      done
+      applyall "${mfdir}/roles"/{pod-nanny,kubernetes-dashboard}".yaml"
+
+      applyall "${mfdir}/role-bindings"/{heapster-nanny,kubernetes-dashboard,metrics-server}".yaml"
 
       {{ if .Experimental.TLSBootstrap.Enabled }}
-      for manifest in {node-bootstrapper,kubelet-certificate-bootstrap}; do
-          kubectl apply -f "${mfdir}/cluster-roles/$manifest.yaml"
-      done
+      applyall "${mfdir}/cluster-roles"/{node-bootstrapper,kubelet-certificate-bootstrap}".yaml"
 
-      for manifest in {node-bootstrapper,kubelet-certificate-bootstrap}; do
-          kubectl apply -f "${mfdir}/cluster-role-bindings/$manifest.yaml"
-      done
+      applyall "${mfdir}/cluster-role-bindings"/{node-bootstrapper,kubelet-certificate-bootstrap}".yaml"
       {{ end }}
 
       {{if .Experimental.Kube2IamSupport.Enabled }}
         mfdir=/srv/kubernetes/manifests
-        kubectl apply -f "${mfdir}/kube2iam-rbac.yaml"
-        kubectl apply -f "${mfdir}/kube2iam-ds.yaml";
+        applyall "${mfdir}/kube2iam-rbac.yaml"
+        applyall "${mfdir}/kube2iam-ds.yaml";
       {{ end }}
 
   - path: /etc/kubernetes/cni/docker_opts_cni.env