Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit b0614a8

Browse files
mumoshumumoshu
authored
WIP: Bump to Kubernetes v1.6.1 (#492)
* Bump to Kubernetes v1.6.1 This change was just the result of running the following commands: ``` $ contrib/bump-version v1.6.1_coreos.0 Updating contrib/bump-version Updating core/controlplane/config/config.go Updating core/controlplane/config/templates/cluster.yaml Updating e2e/kubernetes/Dockerfile Updating e2e/kubernetes/Makefile Updating vendor/github.com/aws/aws-sdk-go/CHANGELOG.md $ git checkout -p -- vendor ``` As etcd3 support is already introduced via #417, after this change is introduced, it was ideally a matter of running E2E against a newly created kube-aws cluster with k8s 1.6.1, which turned out not to be true, hence the subsequent changes. * Use etcd3 by default etcd2 support will be dropped soon, as the etcd3 storage driver is already the default since k8s v.1.6.0. * Bump to calico-cni v1.6.2, which is an even newer release than the one included in the latest calico v2.1.2, to deal with kubernetes/kubernetes#43488 * Set up /etc/kubernetes/cni/net.d not using calico-cni but by our own to deal with kubernetes/kubernetes#43014 * Set up /opt/cni/bin using docker rather than a k8s static pod to prevent temporary "failed to find plugin * in path" errors from cni They were emitted when pods are scheduled but /opt/cni/bin is not yet populated ``` Error syncing pod, skipping: failed to "CreatePodSandbox" for "kube-dns-3816048056-cwx62_kube-system(12c3204f-1a54-11e7-bfb0-06751e989ae7)" with CreatePodSandboxError: "CreatePodSandbox for pod \"kube-dns-3816048056-cwx62_kube-system(12c3204f-1a54-11e7-bfb0-06751e989ae7)\" failed: rpc error: code = 2 desc = NetworkPlugin cni failed to set up pod \"kube-dns-3816048056-cwx62_kube-system\" network: failed to find plugin \"loopback\" in path [/opt/loopback/bin /opt/cni/bin]" ``` * Fix a bug that resulted etcd-member.service to use the default version number 3.0.x regardless of what is specified via `etcd.version` in cluster.yaml. The bug was reported in #497 (comment) * Simplify EtcdVersion func According to the review comment #492 (review) * Fix permanent errors like "failed to find plugin * in path" from cni which was breaking cni + flannel/calico in k8s 1.6, by specifying the `--cni-bin-dir=/opt/cni/bin` flag for kubelets The default dir had been accidentally changed at least in k8s 1.6.0 and 1.6.1. Resolves #494 Resolves #495 E2E against a cluster with flannel passed after this change: ``` $ ETCD_VERSION=3 ETCD_SNAPSHOT_AUTOMATED=1 ETCD_DISASTER_RECOVERY_AUTOMATED=1 ETCD_COUNT=3 KUBE_AWS_CLUSTER_NAME=kubeaws2 ./run all *snip* Ran 151 of 588 Specs in 3492.050 seconds SUCCESS! -- 151 Passed | 0 Failed | 0 Pending | 437 Skipped PASS Ginkgo ran 1 suite in 58m12.359210255s Test Suite Passed 2017/04/04 09:35:29 util.go:127: Step './hack/ginkgo-e2e.sh --ginkgo.focus=\[Conformance\]' finished in 58m12.683100213s 2017/04/04 09:35:29 e2e.go:80: Done ``` Also passed against a cluster with calico: ``` Ran 151 of 588 Specs in 3381.108 seconds SUCCESS! -- 151 Passed | 0 Failed | 0 Pending | 437 Skipped PASS Ginkgo ran 1 suite in 56m21.415087252s Test Suite Passed 2017/04/06 03:58:20 util.go:131: Step './hack/ginkgo-e2e.sh --ginkgo.focus=\[Conformance\]' finished in 56m21.76726736s 2017/04/06 03:58:20 e2e.go:80: Done ```
1 parent 16168b7 commit b0614a8

File tree

12 files changed

+116
-50
lines changed

12 files changed

+116
-50
lines changed

‎contrib/bump-version

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66

77
if [ $# -ne 1 ] || [ `expr $1 : ".*_.*"` == 0 ]; then
88
echo "USAGE: $0 <target-version>"
9-
echo " example: $0 'v1.5.5_coreos.0'"
9+
echo " example: $0 'v1.6.1_coreos.0'"
1010
exit 1
1111
fi
1212

13-
CURRENT_VERSION=${CURRENT_VERSION:-"v1.5.5_coreos.0"}
13+
CURRENT_VERSION=${CURRENT_VERSION:-"v1.6.1_coreos.0"}
1414
TARGET_VERSION=${1}
1515

1616
CURRENT_VERSION_BASE=${CURRENT_VERSION%%_*}

‎core/controlplane/config/config.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import (
2727
)
2828

2929
const (
30-
k8sVer = "v1.5.5_coreos.0"
30+
k8sVer = "v1.6.1_coreos.0"
3131

3232
credentialsDir = "credentials"
3333
userDataDir = "userdata"
@@ -105,7 +105,7 @@ func NewDefaultCluster() *Cluster {
105105
HyperkubeImage: model.Image{Repo: "quay.io/coreos/hyperkube", Tag: k8sVer, RktPullDocker: false},
106106
AWSCliImage: model.Image{Repo: "quay.io/coreos/awscli", Tag: "master", RktPullDocker: false},
107107
CalicoNodeImage: model.Image{Repo: "quay.io/calico/node", Tag: "v1.1.0", RktPullDocker: false},
108-
CalicoCniImage: model.Image{Repo: "quay.io/calico/cni", Tag: "v1.6.1", RktPullDocker: false},
108+
CalicoCniImage: model.Image{Repo: "quay.io/calico/cni", Tag: "v1.6.2", RktPullDocker: false},
109109
CalicoPolicyControllerImage: model.Image{Repo: "quay.io/calico/kube-policy-controller", Tag: "v0.5.4", RktPullDocker: false},
110110
ClusterAutoscalerImage: model.Image{Repo: "gcr.io/google_containers/cluster-proportional-autoscaler-amd64", Tag: "1.0.0", RktPullDocker: false},
111111
KubeDnsImage: model.Image{Repo: "gcr.io/google_containers/kubedns-amd64", Tag: "1.9", RktPullDocker: false},

‎core/controlplane/config/templates/cloud-config-controller

Lines changed: 53 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -165,9 +165,14 @@ coreos:
165165
cluster-health
166166

167167
ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /srv/kubernetes/manifests -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'"
168+
{{if .UseCalico -}}
169+
ExecStartPre=/usr/bin/docker run --rm -e SLEEP=false -v /opt/cni/bin:/host/opt/cni/bin {{ .CalicoCniImage.RepoWithTag }} /install-cni.sh
170+
{{end -}}
168171
ExecStart=/usr/lib/coreos/kubelet-wrapper \
169172
--api-servers=http://localhost:8080 \
170173
--cni-conf-dir=/etc/kubernetes/cni/net.d \
174+
{{/* Work-around until https://github.com/kubernetes/kubernetes/issues/43967 is fixed via https://github.com/kubernetes/kubernetes/pull/43995 */ -}}
175+
--cni-bin-dir=/opt/cni/bin \
171176
--network-plugin={{.K8sNetworkPlugin}} \
172177
--container-runtime={{.ContainerRuntime}} \
173178
--rkt-path=/usr/bin/rkt \
@@ -628,10 +633,14 @@ write_files:
628633
k8s-app: calico-node
629634
annotations:
630635
scheduler.alpha.kubernetes.io/critical-pod: ''
631-
scheduler.alpha.kubernetes.io/tolerations: |
632-
[{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
633-
{"key":"CriticalAddonsOnly", "operator":"Exists"}]
634636
spec:
637+
tolerations:
638+
- key: "node.alpha.kubernetes.io/role"
639+
operator: "Equal"
640+
value: "master"
641+
effect: "NoSchedule"
642+
- key: "CriticalAddonsOnly"
643+
operator: "Exists"
635644
hostNetwork: true
636645
containers:
637646
- name: calico-node
@@ -677,30 +686,6 @@ write_files:
677686
- mountPath: /etc/resolv.conf
678687
name: dns
679688
readOnly: true
680-
- name: install-cni
681-
image: {{ .CalicoCniImage.RepoWithTag }}
682-
imagePullPolicy: Always
683-
command: ["/install-cni.sh"]
684-
env:
685-
- name: ETCD_ENDPOINTS
686-
valueFrom:
687-
configMapKeyRef:
688-
name: calico-config
689-
key: etcd_endpoints
690-
- name: CNI_NETWORK_CONFIG
691-
valueFrom:
692-
configMapKeyRef:
693-
name: calico-config
694-
key: cni_network_config
695-
- name: CNI_NET_DIR
696-
value: "/etc/kubernetes/cni/net.d"
697-
volumeMounts:
698-
- mountPath: /host/opt/cni/bin
699-
name: cni-bin-dir
700-
- mountPath: /host/etc/cni/net.d
701-
name: cni-net-dir
702-
- mountPath: /calico-secrets
703-
name: etcd-certs
704689
volumes:
705690
- name: lib-modules
706691
hostPath:
@@ -732,9 +717,6 @@ write_files:
732717
k8s-app: calico-policy
733718
annotations:
734719
scheduler.alpha.kubernetes.io/critical-pod: ''
735-
scheduler.alpha.kubernetes.io/tolerations: |
736-
[{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
737-
{"key":"CriticalAddonsOnly", "operator":"Exists"}]
738720

739721
spec:
740722
replicas: 1
@@ -745,6 +727,13 @@ write_files:
745727
labels:
746728
k8s-app: calico-policy
747729
spec:
730+
tolerations:
731+
- key: "node.alpha.kubernetes.io/role"
732+
operator: "Equal"
733+
value: "master"
734+
effect: "NoSchedule"
735+
- key: "CriticalAddonsOnly"
736+
operator: "Exists"
748737
hostNetwork: true
749738
containers:
750739
- name: calico-policy-controller
@@ -1262,8 +1251,10 @@ write_files:
12621251
k8s-app: kube-rescheduler
12631252
annotations:
12641253
scheduler.alpha.kubernetes.io/critical-pod: ''
1265-
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
12661254
spec:
1255+
tolerations:
1256+
- key: "CriticalAddonsOnly"
1257+
operator: "Exists"
12671258
hostNetwork: true
12681259
containers:
12691260
- name: kube-rescheduler
@@ -1291,8 +1282,10 @@ write_files:
12911282
k8s-app: kube-dns-autoscaler
12921283
annotations:
12931284
scheduler.alpha.kubernetes.io/critical-pod: ''
1294-
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
12951285
spec:
1286+
tolerations:
1287+
- key: "CriticalAddonsOnly"
1288+
operator: "Exists"
12961289
containers:
12971290
- name: autoscaler
12981291
image: {{ .ClusterAutoscalerImage.RepoWithTag }}
@@ -1338,8 +1331,10 @@ write_files:
13381331
k8s-app: kube-dns
13391332
annotations:
13401333
scheduler.alpha.kubernetes.io/critical-pod: ''
1341-
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
13421334
spec:
1335+
tolerations:
1336+
- key: "CriticalAddonsOnly"
1337+
operator: "Exists"
13431338
containers:
13441339
- name: kubedns
13451340
image: {{ .KubeDnsImage.RepoWithTag }}
@@ -1501,8 +1496,10 @@ write_files:
15011496
version: v1.3.0
15021497
annotations:
15031498
scheduler.alpha.kubernetes.io/critical-pod: ''
1504-
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
15051499
spec:
1500+
tolerations:
1501+
- key: "CriticalAddonsOnly"
1502+
operator: "Exists"
15061503
containers:
15071504
- image: {{ .HeapsterImage.RepoWithTag }}
15081505
name: heapster
@@ -1594,8 +1591,10 @@ write_files:
15941591
kubernetes.io/cluster-service: "true"
15951592
annotations:
15961593
scheduler.alpha.kubernetes.io/critical-pod: ''
1597-
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
15981594
spec:
1595+
tolerations:
1596+
- key: "CriticalAddonsOnly"
1597+
operator: "Exists"
15991598
containers:
16001599
- name: kubernetes-dashboard
16011600
image: {{ .KubeDashboardImage.RepoWithTag }}
@@ -1680,6 +1679,25 @@ write_files:
16801679

16811680
{{ else }}
16821681

1682+
- path: /etc/kubernetes/cni/net.d/10-calico.conf
1683+
content: |
1684+
{
1685+
"name": "calico",
1686+
"type": "flannel",
1687+
"delegate": {
1688+
"type": "calico",
1689+
"etcd_endpoints": "#ETCD_ENDPOINTS#",
1690+
"etcd_key_file": "/etc/kubernetes/ssl/etcd-client-key.pem",
1691+
"etcd_cert_file": "/etc/kubernetes/ssl/etcd-client.pem",
1692+
"etcd_ca_cert_file": "/etc/kubernetes/ssl/ca.pem",
1693+
"log_level": "info",
1694+
"policy": {
1695+
"type": "k8s",
1696+
"k8s_api_root": "http://127.0.0.1:8080/api/v1/"
1697+
}
1698+
}
1699+
}
1700+
16831701
# http://docs.projectcalico.org/v2.0/usage/configuration/
16841702
- path: /etc/modules-load.d/nf.conf
16851703
content: |

‎core/controlplane/config/templates/cloud-config-etcd

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ coreos:
164164
ExecStartPre=/usr/bin/systemctl is-active decrypt-assets.service
165165
{{- end}}
166166
ExecStartPre=/usr/bin/chown -R etcd:etcd /var/lib/etcd2
167+
{{if .Etcd.Version.Is3 }}
168+
- name: 40-version.conf
169+
content: |
170+
[Service]
171+
Environment="ETCD_IMAGE_TAG=v{{.Etcd.Version}}"
172+
{{end}}
167173
enable: true
168174
command: start
169175

‎core/controlplane/config/templates/cloud-config-worker

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,14 @@ coreos:
160160
--cert-file /etc/kubernetes/ssl/etcd-client.pem \
161161
--endpoints "${ETCD_ENDPOINTS}" \
162162
cluster-health
163+
{{if .UseCalico -}}
164+
ExecStartPre=/usr/bin/docker run --rm -e SLEEP=false -v /opt/cni/bin:/host/opt/cni/bin {{ .CalicoCniImage.RepoWithTag }} /install-cni.sh
165+
{{end -}}
163166
ExecStart=/usr/lib/coreos/kubelet-wrapper \
164167
--api-servers={{.APIServerEndpoint}} \
165-
--network-plugin-dir=/etc/kubernetes/cni/net.d \
168+
--cni-conf-dir=/etc/kubernetes/cni/net.d \
169+
{{/* Work-around until https://github.com/kubernetes/kubernetes/issues/43967 is fixed via https://github.com/kubernetes/kubernetes/pull/43995 */ -}}
170+
--cni-bin-dir=/opt/cni/bin \
166171
--network-plugin={{.K8sNetworkPlugin}} \
167172
--container-runtime={{.ContainerRuntime}} \
168173
--rkt-path=/usr/bin/rkt \
@@ -811,6 +816,28 @@ write_files:
811816

812817
{{ else }}
813818

819+
- path: /etc/kubernetes/cni/net.d/10-calico.conf
820+
content: |
821+
{
822+
"name": "calico",
823+
"type": "flannel",
824+
"delegate": {
825+
"type": "calico",
826+
"etcd_endpoints": "#ETCD_ENDPOINTS#",
827+
"etcd_key_file": "/etc/kubernetes/ssl/etcd-client-key.pem",
828+
"etcd_cert_file": "/etc/kubernetes/ssl/etcd-client.pem",
829+
"etcd_ca_cert_file": "/etc/kubernetes/ssl/ca.pem",
830+
"log_level": "info",
831+
"policy": {
832+
"type": "k8s",
833+
"k8s_api_root": "https://{{.APIEndpoint.DNSName}}/api/v1/",
834+
"k8s_client_key": "/etc/kubernetes/ssl/worker-key.pem",
835+
"k8s_client_certificate": "/etc/kubernetes/ssl/worker.pem",
836+
"k8s_certificate_authority": "/etc/kubernetes/ssl/ca.pem"
837+
}
838+
}
839+
}
840+
814841
# http://docs.projectcalico.org/v2.0/usage/configuration/
815842
- path: /etc/modules-load.d/nf.conf
816843
content: |

‎core/controlplane/config/templates/cluster.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -801,7 +801,7 @@ worker:
801801
# and pulling from quay or dockerhub is slow and you get many timeouts.
802802

803803
# Version of hyperkube image to use. This is the tag for the hyperkube image repository.
804-
# kubernetesVersion: v1.5.5_coreos.0
804+
# kubernetesVersion: v1.6.1_coreos.0
805805

806806
# Hyperkube image repository to use.
807807
# hyperkubeImage:

‎core/controlplane/config/templates/stack-template.json

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -884,6 +884,23 @@
884884
},
885885
"Type": "AWS::EC2::SecurityGroup"
886886
},
887+
{{if $.UseCalico -}}
888+
{{/* Required by calico-policy-controller when calico is enabled. See https://github.com/kubernetes-incubator/kube-aws/issues/494#issuecomment-291687137 */}}
889+
"SecurityGroupControllerIngressFromControllerToController": {
890+
"Properties": {
891+
"FromPort": 443,
892+
"GroupId": {
893+
"Ref": "SecurityGroupController"
894+
},
895+
"IpProtocol": "tcp",
896+
"SourceSecurityGroupId": {
897+
"Ref": "SecurityGroupController"
898+
},
899+
"ToPort": 443
900+
},
901+
"Type": "AWS::EC2::SecurityGroupIngress"
902+
},
903+
{{end -}}
887904
"SecurityGroupControllerIngressFromControllerToKubelet": {
888905
"Properties": {
889906
"FromPort": 10250,

‎core/nodepool/config/deployment.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ func (c DeploymentSettings) WithDefaultsFrom(main cfg.DeploymentSettings) Deploy
9595
c.HyperkubeImage.Tag = c.K8sVer
9696
c.AWSCliImage.MergeIfEmpty(main.AWSCliImage)
9797
c.CalicoCtlImage.MergeIfEmpty(main.CalicoCtlImage)
98+
c.CalicoCniImage.MergeIfEmpty(main.CalicoCniImage)
9899
c.PauseImage.MergeIfEmpty(main.PauseImage)
99100
c.FlannelImage.MergeIfEmpty(main.FlannelImage)
100101

‎e2e/kubernetes/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM golang:1.7.1
22

3-
ARG KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.5.5+coreos.0}
3+
ARG KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.6.1+coreos.0}
44

55
RUN apt-get update && \
66
apt-get install -y rsync && \

‎e2e/kubernetes/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
KUBERNETES_VERSION ?= v1.5.5+coreos.0
1+
KUBERNETES_VERSION ?= v1.6.1+coreos.0
22
DOCKER_REPO ?=
33
DOCKER_TAG ?= $(DOCKER_REPO)kube-e2e:$(KUBERNETES_VERSION)
44
DOCKER_TAG_SANITIZED ?= $(shell echo $(DOCKER_TAG) | sed -e 's/+/_/')

‎e2e/run

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ SRC_DIR=$(cd $(dirname $0); cd ..; pwd)
99
KUBECONFIG=${WORK_DIR}/kubeconfig
1010
ETCD_COUNT=${ETCD_COUNT:-3}
1111
CONTROLLER_COUNT=${CONTROLLER_COUNT:-2}
12-
ETCD_VERSION=${ETCD_VERSION:-3}
12+
ETCD_VERSION=${ETCD_VERSION:-}
1313

1414
export KUBECONFIG
1515

@@ -234,7 +234,7 @@ etcd:
234234
count: $ETCD_COUNT" >> cluster.yaml
235235

236236
if [ "${ETCD_VERSION}" != "" ]; then
237-
echo -e " version: 3" >> cluster.yaml
237+
echo -e " version: ${ETCD_VERSION}" >> cluster.yaml
238238
fi
239239

240240
if [ "${ETCD_DISASTER_RECOVERY_AUTOMATED}" != "" ]; then

‎model/etcd.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,15 +129,12 @@ func (e Etcd) SystemdUnitName() string {
129129
return "etcd2.service"
130130
}
131131

132-
// Version returns the version of etcd (e.g. `2`, `3`, `3.1.3`) to be used for this etcd cluster
132+
// Version returns the version of etcd (e.g. `3.1.5`) to be used for this etcd cluster
133133
func (e Etcd) Version() EtcdVersion {
134134
if e.Cluster.Version != "" {
135135
return e.Cluster.Version
136136
}
137-
if e.Cluster.Version == "3" {
138-
return "3.1.3"
139-
}
140-
return "2"
137+
return "3.1.5"
141138
}
142139

143140
func (v EtcdVersion) Is3() bool {

0 commit comments

Comments
 (0)