Merge pull request #26 from dweomer/testing/sonobuoy

testing: sonobuoy plugin for e2e testing

Author: dweomer

.drone.yml

@@ -11,11 +11,12 @@ steps:
   image: rancher/dapper:v0.4.2
   commands:
   - dapper ci
+  - dapper e2e-verify
   volumes:
   - name: docker
     path: /var/run/docker.sock
 
-- name: github_binary_release
+- name: upload-artifacts
   image: ibuildthecloud/github-release:v0.0.1
   settings:
     api_key:
@@ -36,7 +37,7 @@ steps:
     event:
     - tag
 
-- name: docker-publish
+- name: push-controller
   image: plugins/docker
   settings:
     dockerfile: package/Dockerfile
@@ -55,6 +56,26 @@ steps:
     event:
     - tag
 
+- name: push-e2e-tests
+  image: plugins/docker
+  settings:
+    dockerfile: package/Dockerfile
+    target: e2e-tests
+    password:
+      from_secret: docker_password
+    repo: "rancher/system-upgrade-controller"
+    tag: "${DRONE_TAG}-amd64-e2e-tests"
+    username:
+      from_secret: docker_username
+  when:
+    instance:
+      - drone-publish.rancher.io
+    ref:
+      - refs/head/master
+      - refs/tags/*
+    event:
+      - tag
+
 volumes:
 - name: docker
   host:
@@ -77,7 +98,7 @@ steps:
   - name: docker
     path: /var/run/docker.sock
 
-- name: github_binary_release
+- name: upload-artifacts
   image: ibuildthecloud/github-release:v0.0.1
   settings:
     api_key:
@@ -98,7 +119,7 @@ steps:
     event:
     - tag
 
-- name: docker-publish
+- name: push-controller
   image: plugins/docker
   settings:
     dockerfile: package/Dockerfile
@@ -117,6 +138,26 @@ steps:
     event:
     - tag
 
+- name: push-e2e-tests
+  image: plugins/docker
+  settings:
+    dockerfile: package/Dockerfile
+    target: e2e-tests
+    password:
+      from_secret: docker_password
+    repo: "rancher/system-upgrade-controller"
+    tag: "${DRONE_TAG}-arm64-e2e-tests"
+    username:
+      from_secret: docker_username
+  when:
+    instance:
+      - drone-publish.rancher.io
+    ref:
+      - refs/head/master
+      - refs/tags/*
+    event:
+      - tag
+
 volumes:
 - name: docker
   host:
@@ -139,7 +180,7 @@ steps:
   - name: docker
     path: /var/run/docker.sock
 
-- name: github_binary_release
+- name: upload-artifacts
   image: ibuildthecloud/github-release:v0.0.1
   settings:
     api_key:
@@ -160,7 +201,7 @@ steps:
     event:
     - tag
 
-- name: docker-publish
+- name: push-controller
   image: plugins/docker
   settings:
     dockerfile: package/Dockerfile
@@ -193,7 +234,7 @@ platform:
   arch: amd64
 
 steps:
-- name: manifest
+- name: controller
   image: plugins/manifest:1.0.2
   settings:
     username:
@@ -215,6 +256,27 @@ steps:
     event:
     - tag
 
+- name: e2e-tests
+  image: plugins/manifest:1.0.2
+  settings:
+    username:
+      from_secret: docker_username
+    password:
+      from_secret: docker_password
+    platforms:
+      - linux/amd64
+      - linux/arm64
+    target: "rancher/system-upgrade-controller:${DRONE_TAG}-e2e-tests"
+    template: "rancher/system-upgrade-controller:${DRONE_TAG}-ARCH-e2e-tests"
+  when:
+    instance:
+      - drone-publish.rancher.io
+    ref:
+      - refs/head/master
+      - refs/tags/*
+    event:
+      - tag
+
 depends_on:
 - amd64
 - arm64

Dockerfile.dapper

@@ -1,9 +1,17 @@
+ARG KUBECTL=rancher/kubectl:v1.17.0
+# ARG SONOBUOY=sonobuoy/sonobuoy:v0.17.2
+# FROM ${SONOBUOY} AS sonobuoy
+FROM ${KUBECTL} AS kubectl
+
 FROM golang:1.13-alpine
 
+COPY --from=kubectl /bin/kubectl /usr/local/bin/kubectl
+# COPY --from=sonobuoy /sonobuoy /usr/local/bin/sonobuoy
+
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
 
-RUN apk -U add bash expect git gcc jq musl-dev docker vim less file curl wget ca-certificates
+RUN apk -U add coreutils bash expect git gcc jq musl-dev docker docker-compose vim less file curl wget ca-certificates iproute2
 RUN go get -d golang.org/x/lint/golint && \
     git -C /go/src/golang.org/x/lint/golint checkout -b current 06c8688daad7faa9da5a0c2f163a3d14aac986ca && \
     go install golang.org/x/lint/golint && \
@@ -16,13 +24,21 @@ RUN rm -rf /go/src /go/pkg
 RUN if [ "${ARCH}" == "amd64" ]; then \
         curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.15.0; \
     fi
+ARG SONOBUOY_VERSION=0.17.2
+RUN if [ "${ARCH}" != "arm" ]; then \
+        set -x; \
+        curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${ARCH}.tar.gz" \
+        | tar -xz -C /usr/local/bin; \
+        chmod +x /usr/local/bin/sonobuoy; \
+    fi
 
 # ENV DAPPER_RUN_ARGS --privileged
 ENV DAPPER_ENV REPO TAG DRONE_TAG
 ENV DAPPER_SOURCE /go/src/github.com/rancher/system-upgrade-controller/
 ENV DAPPER_OUTPUT ./bin ./dist
 ENV DAPPER_DOCKER_SOCKET true
 ENV HOME ${DAPPER_SOURCE}
+ENV KUBECONFIG /root/.kube/config
 ENV GO111MODULE off
 WORKDIR ${DAPPER_SOURCE}
 

Makefile

@@ -10,9 +10,11 @@ TARGETS := $(shell ls scripts)
 $(TARGETS): .dapper
 	./.dapper $@
 
-e2e:
-	./scripts/verify
+e2e: e2e-verify
+
+clean:
+	rm -rvf ./bin ./dist
 
 .DEFAULT_GOAL := ci
 
-.PHONY: $(TARGETS) e2e
+.PHONY: $(TARGETS) e2e clean

README.md

@@ -52,7 +52,6 @@ spec:
 
   # The value for `channel` is assumed to be a URL that returns HTTP 302 with the last path element of the value
   # returned in the Location header assumed to be an image tag.
-  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L177
   channel: https://github.com/rancher/k3os/releases/latest
 
   # Providing a value for `version` will prevent polling/resolution of the `channel` if specified.
@@ -63,10 +62,8 @@ spec:
     matchExpressions:
       # This limits application of this upgrade only to nodes that have opted in by applying this label.
       # Additionally, a value of `disabled` for this label on a node will cause the controller to skip over the node.
-      # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L216
       # NOTICE THAT THE NAME PORTION OF THIS LABEL MATCHES THE PLAN NAME. This is related to the fact that the
-      # system-upgrade-controller will tag the node with this very label having the value of the applied version.
-      # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L112-L115
+      # system-upgrade-controller will tag the node with this very label having the value of the applied plan.status.latestHash.
       - {key: plan.upgrade.cattle.io/k3os-latest, operator: Exists}
       # This label is set by k3OS, therefore a node without it should not apply this upgrade.
       - {key: k3os.io/mode, operator: Exists}
@@ -83,15 +80,13 @@ spec:
   #   image: alpine:3.11
   #   command: [sh, -c]
   #   args: [" echo '### ENV ###'; env | sort; echo '### RUN ###'; find /run/system-upgrade | sort"]
-  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L58
 
   # See https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/#use-kubectl-drain-to-remove-a-node-from-service
   drain:
     # deleteLocalData: true
     # ignoreDaemonSets: true
     force: true
 
-  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L51
   upgrade:
     # The tag portion of the image will be overridden with the value from `.status.latestVersion` a.k.a. the resolved version.
     # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L47
@@ -112,7 +107,10 @@ spec:
 ```
 
 ## Building
-`make`
+
+```shell script
+make
+```
 
 ### Local Execution
 
@@ -122,6 +120,27 @@ Also see [`manifests/system-upgrade-controller.yaml`](manifests/system-upgrade-c
 "typical" deployment might look like with default environment variables that parameterize various operational aspects
 of the controller and the resources spawned by it.
 
+### End-to-End Testing
+
+Integration tests are bundled as a [Sonobuoy plugin](https://sonobuoy.io/docs/v0.17.2/plugins/) that expects to be run within a pod.
+To verify locally:
+
+```shell script
+make e2e
+```
+
+This will, via Dapper, stand up a local cluster (using docker-compose) and then run the Sonobuoy plugin against/within it.
+The Sonobuoy results are parsed and a `Status: passed` results in a clean exit, whereas `Status: failed` exits non-zero.
+See [scripts/e2e-results](scripts/e2e-results) called by [scripts/e2e-verify](scripts/e2e-verify).
+
+Alternatively, if you have a working cluster and Sonobuoy installation, provided you've pushed the images (consider building with
+something like `make REPO=dweomer TAG=dev`), then you can run the e2e tests thusly:
+
+```shell script
+sonobuoy run --plugin dist/artifacts/system-upgrade-controller-e2e-tests.yaml --wait
+sonobuoy results $(sonobuoy retrieve)
+```
+
 ## License
 Copyright (c) 2019-2020 [Rancher Labs, Inc.](http://rancher.com)
 

e2e/cluster/local/docker-compose.yaml

@@ -0,0 +1,41 @@
+version: "3.7"
+
+volumes:
+  kubeconfig:
+    name: e2e-local-kubeconfig
+
+services:
+
+  leader:
+    container_name: local-leader
+    image: "e2e/cluster/local/k3s"
+    build:
+      context: ./images/k3s/.
+      args:
+        - ARCH=${ARCH}
+    command: server --no-deploy=traefik,metrics-server
+    environment:
+      - K3S_TOKEN=e2e
+    hostname: local-leader
+    privileged: true
+    ports:
+      - "172.17.0.1:6443:6443" # k3s
+    volumes:
+      - source: kubeconfig
+        target: /etc/rancher/k3s
+        type: volume
+
+  worker:
+    depends_on:
+      - leader
+    container_name: local-worker
+    image: "e2e/cluster/local/k3s"
+#    build:
+#      context: ./images/k3s/.
+#      args:
+#        - ARCH=${ARCH}
+    command: agent --server https://local-leader:6443
+    environment:
+      - K3S_TOKEN=e2e
+    hostname: local-worker
+    privileged: true

e2e/cluster/local/images/k3s/Dockerfile

@@ -0,0 +1,29 @@
+# Install the all-in-one binary so we can copy our run-time images into the image
+# which helps avoid pulling them when running e2e tests.
+ARG ALPINE="library/alpine:3.11"
+FROM ${ALPINE} AS k3s
+ARG ARCH
+ARG K3S_VERSION="v1.17.2+k3s1"
+RUN set -x \
+ && apk --no-cache add \
+    bash \
+    ca-certificates \
+    curl \
+ && if [ "${ARCH?required}" != "amd64" ]; then \
+        K3S_SUFFIX="-${ARCH}"; \
+    fi \
+ && curl -fsSL "https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s${K3S_SUFFIX}" > /bin/k3s \
+ && chmod +x /bin/k3s \
+ && ln -s /bin/k3s /bin/kubectl \
+ && ln -s /bin/k3s /bin/ctr \
+ && k3s --version
+
+COPY scratch/*-${ARCH}.tar /var/lib/rancher/k3s/agent/images/
+
+VOLUME /var/lib/cni
+VOLUME /var/lib/kubelet
+VOLUME /var/lib/rancher/k3s
+VOLUME /var/log
+
+ENTRYPOINT ["k3s"]
+CMD ["server"]

e2e/cluster/local/images/k3s/scratch/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

e2e/cluster/local/scripts/cluster-down

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd $(dirname $0)/..
+
+docker-compose down \
+  --remove-orphans \
+  --rmi local \
+  --volumes

e2e/cluster/local/scripts/cluster-get-kubeconfig

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -e -o pipefail
+
+cd $(dirname $0)/..
+
+mkdir -vp "$(dirname ${KUBECONFIG:="${HOME}/.kube/config"})"
+docker exec local-leader kubectl config view --raw | sed -e 's/127.0.0.1/172.17.0.1/g' > "${KUBECONFIG}"
+#echo "${KUBECONFIG}" >&2

e2e/cluster/local/scripts/cluster-num-nodes

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+docker ps \
+  --filter label=com.docker.compose.project=local \
+  --format='{{.Image}} ' \
+| grep 'e2e/cluster/local/k3s' | wc -l | awk '{print $1}' | tr -d '[:space:]'