Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit c186253

Browse files
mumoshumumoshu
authored
Merge pull request #5 from mumoshu/easier-conformance-testing
feat(test): Implement an easier way to run k8s conformance tests against kube-aws-provisioned cluster
2 parents 3805008 + 8d6d0a4 commit c186253

File tree

6 files changed

+308
-0
lines changed

6 files changed

+308
-0
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@
44
*~
55
/config/templates.go
66
.idea/
7+
.envrc

‎e2e/README.md

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# End-to-end testing for kube-aws
2+
3+
This directory contains a set of tools to run end-to-end testing for kube-aws.
4+
It is composed of:
5+
6+
* Cluster creation using `kube-aws`
7+
* [Kubernetes Conformance Tests](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/e2e-tests.md#conformance-tests)
8+
9+
To run e2e tests, you should have set all he required env vars.
10+
For convenience, creating `.envrc` used by `direnv` like as follows would be good.
11+
12+
```
13+
export KUBE_AWS_KEY_NAME=...
14+
export KUBE_AWS_KMS_KEY_ARN=arn:aws:kms:us-west-1:<account id>:key/<id>
15+
export KUBE_AWS_DOMAIN=example.com
16+
export KUBE_AWS_REGION=us-west-1
17+
export KUBE_AWS_AVAILABILITY_ZONE=us-west-1b
18+
export KUBE_AWS_HOSTED_ZONE_ID=...
19+
20+
export DOCKER_REPO=quay.io/mumoshu/
21+
export SSH_PRIVATE_KEY=path/to/private/key/matching/key/name
22+
```
23+
24+
Finally, run e2e tests like:
25+
26+
```
27+
$ KUBE_AWS_CLUSTER_NAME=kubeawstest ./run all
28+
```

‎e2e/kubernetes/Dockerfile

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
FROM golang:1.7.1
2+
3+
ARG KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.4.3+coreos.0}
4+
5+
RUN apt-get update && \
6+
apt-get install -y rsync && \
7+
go get -u github.com/jteeuwen/go-bindata/go-bindata && \
8+
rm -rf /var/lib/apt/lists/*
9+
10+
WORKDIR /go
11+
12+
RUN git clone --depth 1 https://github.com/coreos/kubernetes -b $KUBERNETES_VERSION && \
13+
cd kubernetes && \
14+
git branch && \
15+
git checkout -b $KUBERNETES_VERSION && \
16+
git branch && \
17+
make all WHAT=cmd/kubectl && \
18+
find . -type f -executable -name kubectl && \
19+
./_output/local/bin/linux/amd64/kubectl version --client && \
20+
make all WHAT=vendor/github.com/onsi/ginkgo/ginkgo && \
21+
make all WHAT=test/e2e/e2e.test && \
22+
rm -Rf .git
23+
24+
ENV KUBERNETES_PROVIDER skeleton
25+
ENV KUBERNETES_CONFORMANCE_TEST Y
26+
27+
ADD conformance.sh /
28+
29+
WORKDIR /go/kubernetes
30+
31+
CMD /conformance.sh

‎e2e/kubernetes/Makefile

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
KUBERNETES_VERSION ?= v1.4.3+coreos.0
2+
DOCKER_REPO ?=
3+
DOCKER_TAG ?= $(DOCKER_REPO)kube-e2e:$(KUBERNETES_VERSION)
4+
DOCKER_TAG_SANITIZED ?= $(shell echo $(DOCKER_TAG) | sed -e 's/+/_/')
5+
KUBE_AWS_ASSETS ?= /missing-kube-aws-assets
6+
SSH_PRIVATE_KEY ?=
7+
SSH_CMD ?= ssh -i $(SSH_PRIVATE_KEY)
8+
MASTER_HOST ?=
9+
FOCUS ?= \[Conformance\]
10+
11+
.PHONY: build
12+
build:
13+
docker build --build-arg KUBERNETES_VERSION=$(KUBERNETES_VERSION) -t $(DOCKER_TAG_SANITIZED) .
14+
15+
.PHONY: publish
16+
publish:
17+
docker push $(DOCKER_TAG_SANITIZED)
18+
19+
.PHONY: run-locally
20+
run-locally:
21+
docker run --rm -v $(KUBE_AWS_ASSETS):/kube $(DOCKER_TAG_SANITIZED)
22+
23+
.PHONY: ssh
24+
ssh:
25+
ssh -i $(SSH_PRIVATE_KEY) core@$(MASTER_HOST)
26+
27+
# example usage:
28+
# FOCUS='\[HPA\].*\[Conformance\]' DOCKER_REPO=quay.io/mumoshu/ MASTER_HOST=**.**.**.** SSH_PRIVATE_KEY=path/to/key KUBE_AWS_ASSETS=path/to/where/kube-aws/run sh -c 'make build && make publish && make run-remotely'
29+
#
30+
# FOCUS:
31+
# a regexp to selectively run e2e tests. Default: \[Conformance\]
32+
# see https://github.com/kubernetes/kubernetes/blob/master/docs/devel/e2e-tests.md#conformance-tests for more examples and description
33+
.PHONY: run-remotely
34+
run-remotely:
35+
rsync -a -e 'ssh -i $(SSH_PRIVATE_KEY)' $(KUBE_AWS_ASSETS)/ core@$(MASTER_HOST):/home/core/kube
36+
ssh -i $(SSH_PRIVATE_KEY) core@$(MASTER_HOST) 'sh -c "docker pull $(DOCKER_TAG_SANITIZED) && container_id=$$(docker run -d -v /home/core/kube:/kube -e FOCUS='$(FOCUS)' $(DOCKER_TAG_SANITIZED)) && echo \$$container_id && docker logs -f \$$container_id"'
37+
38+
.PHONY: get-log
39+
get-log:
40+
full_container_id=$$($(SSH_CMD) core@$(MASTER_HOST) 'docker ps -a --no-trunc | grep $(DOCKER_TAG_SANITIZED) | tee /dev/stderr | head -n 1 | awk "{ print \$$1 }"') && \
41+
rsync --rsync-path="sudo rsync" -e '$(SSH_CMD)' core@$(MASTER_HOST):/var/lib/docker/containers/$${full_container_id}/$${full_container_id}-json.log .
42+
43+
.PHONY: show-log
44+
show-log:
45+
$(SSH_CMD) core@$(MASTER_HOST) 'sh -c "container_id=$$(docker ps -a | grep $(DOCKER_TAG_SANITIZED) | head -n 1 | cut -d" " -f 1); docker logs -f \$$container_id"'
46+
47+
.PHONY: sh
48+
sh:
49+
docker run -it --rm -v $(KUBE_AWS_ASSETS):/kube $(DOCKER_TAG_SANITIZED) sh

‎e2e/kubernetes/conformance.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
set -eu
4+
5+
if [ ! -d /kube ]; then
6+
echo /kube does not exist. run this docker container like '`docker run -v path-to-kube-assets:/kube image-name`' 1>&2
7+
exit 1
8+
fi
9+
10+
DIR=/kube-temp
11+
12+
mkdir ${DIR}
13+
14+
cp -R /kube/* ${DIR}/
15+
16+
export KUBECONFIG=${DIR}/kubeconfig
17+
18+
sed -i -e "s#credentials/#${DIR}/credentials/#g" ${KUBECONFIG}
19+
20+
set -vx
21+
22+
if [ "$FOCUS" != "" ]; then
23+
FOCUS=$(echo $FOCUS | sed -e 's/\[/\\[/' -e 's/\]/\\]/')
24+
fi
25+
26+
FOCUS=${FOCUS:-\[Conformance\]}
27+
28+
go run hack/e2e.go -v --test -check_version_skew=false -check_node_count=true --test_args="--ginkgo.focus=$FOCUS"

‎e2e/run

+171
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
#!/bin/bash
2+
3+
KUBE_AWS_CMD=${KUBE_AWS_CMD:-$GOPATH/src/github.com/coreos/kube-aws/bin/kube-aws}
4+
E2E_DIR=$(cd $(dirname $0); pwd)
5+
WORK_DIR=${E2E_DIR}/assets/${KUBE_AWS_CLUSTER_NAME}
6+
KUBECONFIG=${WORK_DIR}/kubeconfig
7+
8+
export KUBECONFIG
9+
10+
USAGE_EXAMPLE="KUBE_AWS_CLUSTER_NAME=kubeawstest1 KUBE_AWS_KEY_NAME=name/of/ec2/key KUBE_AWS_KMS_KEY_ARN=arn:aws:kms:us-west-1:<account id>:key/your-key KUBE_AWS_REGION=us-west-1 KUBE_AWS_AVAILABILITY_ZONE=us-west-1b ./$0 [prepare|configure|start|all]"
11+
12+
if [ "${KUBE_AWS_CLUSTER_NAME}" == "" ]; then
13+
echo KUBE_AWS_CLUSTER_NAME is not set. Run this command like $USAGE_EXAMPLE 1>&2
14+
exit 1
15+
fi
16+
17+
if [ "${KUBE_AWS_KEY_NAME}" == "" ]; then
18+
echo KUBE_AWS_KEY_NAME is not set. Run this command like $USAGE_EXAMPLE 1>&2
19+
exit 1
20+
fi
21+
22+
if [ "${KUBE_AWS_REGION}" == "" ]; then
23+
echo KUBE_AWS_REGION is not set. Run this command like $USAGE_EXAMPLE 1>&2
24+
exit 1
25+
fi
26+
27+
if [ "${KUBE_AWS_AVAILABILITY_ZONE}" == "" ]; then
28+
echo KUBE_AWS_REGION is not set. Run this command like $USAGE_EXAMPLE 1>&2
29+
exit 1
30+
fi
31+
32+
if [ ! -e "${KUBE_AWS_CMD}" ]; then
33+
echo ${KUBE_AWS_CMD} does not exist. 1>&2
34+
exit 1
35+
fi
36+
37+
KUBE_AWS_VERSION=$($KUBE_AWS_CMD version)
38+
echo Using the kube-aws command at ${KUBE_AWS_CMD}"($KUBE_AWS_VERSION)". Set KUBE_AWS_CMD=path/to/kube-aws to override.
39+
40+
EXTERNAL_DNS_NAME=${KUBE_AWS_CLUSTER_NAME}.${KUBE_AWS_DOMAIN}
41+
echo The kubernetes API would be accessible via ${EXTERNAL_DNS_NAME}
42+
43+
prepare() {
44+
echo Creating or ensuring existence of the kube-aws assets directory ${WORK_DIR}
45+
mkdir -p ${WORK_DIR}
46+
}
47+
48+
configure() {
49+
cd ${WORK_DIR}
50+
51+
${KUBE_AWS_CMD} init \
52+
--cluster-name ${KUBE_AWS_CLUSTER_NAME} \
53+
--external-dns-name ${EXTERNAL_DNS_NAME} \
54+
--region ${KUBE_AWS_REGION} \
55+
--availability-zone ${KUBE_AWS_AVAILABILITY_ZONE} \
56+
--key-name ${KUBE_AWS_KEY_NAME} \
57+
--kms-key-arn ${KUBE_AWS_KMS_KEY_ARN}
58+
59+
echo "hostedZoneId: ${KUBE_AWS_HOSTED_ZONE_ID}" >> cluster.yaml
60+
echo 'createRecordSet: true' >> cluster.yaml
61+
62+
# required to run kube-aws update
63+
echo 'workerCount: 4' >> cluster.yaml
64+
echo 'controllerCount: 2' >> cluster.yaml
65+
66+
${KUBE_AWS_CMD} render
67+
68+
${KUBE_AWS_CMD} validate
69+
70+
echo Generated configuration files in ${WORK_DIR}:
71+
find .
72+
}
73+
74+
clean() {
75+
cd ${WORK_DIR}/..
76+
if [ -d "${KUBE_AWS_CLUSTER_NAME}" ]; then
77+
echo Removing the directory "${WORK_DIR}"
78+
rm -Rf ./${KUBE_AWS_CLUSTER_NAME}/*
79+
fi
80+
}
81+
82+
up() {
83+
cd ${WORK_DIR}
84+
85+
${KUBE_AWS_CMD} up
86+
87+
printf 'Waiting for the Kubernetes API to be accessible'
88+
89+
while ! kubectl get no 2>/dev/null; do
90+
sleep 10
91+
printf '.'
92+
done
93+
echo done
94+
}
95+
96+
destroy() {
97+
aws cloudformation delete-stack --stack-name ${KUBE_AWS_CLUSTER_NAME}
98+
aws cloudformation wait stack-delete-complete --stack-name ${KUBE_AWS_CLUSTER_NAME}
99+
}
100+
101+
test-upgrade() {
102+
SED_CMD="sed -e 's/workerCount: 2/workerCount: 4/' -e 's/controllerCount: 2/controllerCount: 3/'"
103+
diff --unified cluster.yaml <(cat cluster.yaml | sh -c "${SED_CMD}")
104+
sh -c "${SED_CMD} -i bak cluster.yaml"
105+
${KUBE_AWS_CMD} update
106+
aws cloudformation wait stack-update-complete --stack-name ${KUBE_AWS_CLUSTER_NAME}
107+
}
108+
109+
test-destruction() {
110+
aws cloudformation wait stack-delete-complete --stack-name ${KUBE_AWS_CLUSTER_NAME}
111+
}
112+
113+
# Usage: DOCKER_REPO=quay.io/mumoshu/ SSH_PRIVATE_KEY=path/to/private/key ./e2e run conformance
114+
conformance() {
115+
cd ${E2E_DIR}/kubernetes
116+
117+
if [ "$DOCKER_REPO" == "" ]; then
118+
echo DOCKER_REPO is not set.
119+
exit 1
120+
fi
121+
122+
if [ "$SSH_PRIVATE_KEY" == "" ]; then
123+
echo SSH_PRIVATE_KEY is not set.
124+
exit 1
125+
fi
126+
127+
if [ ! -f "$SSH_PRIVATE_KEY" ]; then
128+
echo ${SSH_PRIVATE_KEY} does not exist.
129+
exit 1
130+
fi
131+
132+
master_host=$(aws ec2 describe-instances --output json --query "Reservations[].Instances[]
133+
| [?Tags[?Key==\`aws:cloudformation:stack-name\`].Value|[0]==\`${KUBE_AWS_CLUSTER_NAME}\`]
134+
| [?Tags[?Key==\`aws:cloudformation:logical-id\`].Value|[0]==\`AutoScaleController\`][]
135+
| []" | jq -r 'map({InstanceId: .InstanceId, PublicIpAddress: .PublicIpAddress}) | first | .PublicIpAddress'
136+
)
137+
138+
echo Connecting to $master_host via SSH
139+
140+
KUBE_AWS_ASSETS=${WORK_DIR} MASTER_HOST=$master_host make run-remotely
141+
}
142+
143+
conformance_result() {
144+
cd ${E2E_DIR}/kubernetes
145+
146+
master_host=$(aws ec2 describe-instances --output json --query "Reservations[].Instances[]
147+
| [?Tags[?Key==\`aws:cloudformation:stack-name\`].Value|[0]==\`${KUBE_AWS_CLUSTER_NAME}\`]
148+
| [?Tags[?Key==\`aws:cloudformation:logical-id\`].Value|[0]==\`AutoScaleController\`][]
149+
| []" | jq -r 'map({InstanceId: .InstanceId, PublicIpAddress: .PublicIpAddress}) | first | .PublicIpAddress'
150+
)
151+
152+
echo Connecting to $master_host via SSH
153+
154+
KUBE_AWS_ASSETS=${WORK_DIR} MASTER_HOST=$master_host make show-log
155+
}
156+
157+
all() {
158+
prepare
159+
configure
160+
up
161+
conformance
162+
}
163+
164+
if [ "$1" == "" ]; then
165+
echo Usage: $USAGE_EXAMPLE
166+
exit 1
167+
fi
168+
169+
set -vxe
170+
171+
"$@"

0 commit comments

Comments
 (0)