build: Add E2E tests

Add new E2E tests that ensure cis-operator mechanics are working
as intented. Note that this does not aim to confirm test accuracy,
as that is security-scan's responsibility.

Signed-off-by: Paulo Gomes <[email protected]>

Author: pjbgf

Dockerfile.dapper

@@ -1,14 +1,26 @@
 FROM registry.suse.com/bci/golang:1.19
 
+# k3d and kubectl versions must be aligned with the Kubernetes versions
+# set in tests/k3s-bench-test.yaml.
+#  k3d is used for e2e tests and is not shipped on the final image.
+ARG K3D_VERSION=5.4.8
+ARG KUBERNETES_VERSION=1.28.0
+
+ENV GOLANGCI_LINT v1.51.2
+
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
-ENV GOLANGCI_LINT v1.51.2
 
-RUN zypper -n install git docker vim less file curl wget
+RUN zypper -n install git docker vim less file curl wget awk jq
 RUN if [[ "${ARCH}" == "amd64" ]]; then \
         curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s ${GOLANGCI_LINT}; \
     fi
 
+RUN curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG="v${K3D_VERSION}" bash
+ADD --chown=root:root --chmod=755 \
+    "https://dl.k8s.io/release/v${KUBERNETES_VERSION}/bin/linux/${ARCH}/kubectl" \
+    /usr/local/bin/kubectl
+
 ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS
 ENV DAPPER_SOURCE /go/src/github.com/rancher/cis-operator/
 ENV DAPPER_OUTPUT ./bin ./dist

package/Dockerfile

@@ -1,6 +1,11 @@
 FROM registry.suse.com/bci/bci-busybox:15.5
 
 COPY pkg/ pkg/
+
+# Ensure 65535 can access the templates in
+# pkg/securityscan/core/templates
+RUN chmod -R +xr pkg/
+
 COPY bin/cis-operator /usr/bin/
 
 USER 65535:65535

scripts/ci

@@ -8,3 +8,4 @@ cd $(dirname $0)
 ./validate
 ./validate-ci
 ./package
+./e2e

scripts/e2e

@@ -0,0 +1,138 @@
+#!/bin/bash
+set -eoux pipefail
+
+export ARCH="${ARCH:-amd64}"
+export IMAGE=cis-operator:e2e
+
+# TODO: dynamically find images required and preload them into k3d.
+export SECURITY_SCAN_IMAGE=rancher/security-scan:v0.2.13
+export SONOBUOY_IMAGE=rancher/mirrored-sonobuoy-sonobuoy:v0.56.16
+export COREDNS_IMAGE=rancher/mirrored-coredns-coredns:1.9.4
+export HELM_IMAGE=rancher/klipper-helm:v0.7.4-build20221121
+
+CLUSTER_NAME="cis-op-e2e-${RANDOM}"
+E2E_TIMEOUT_SECONDS=200
+
+CANCELLING=""
+NETWORK_ID=""
+CURRENT_CONTAINER=$(cat /etc/hostname)
+
+function cleanup() {
+  CANCELLING="true"
+  echo "Cleaning up clusters..."
+  
+  docker stop "k3d-${CLUSTER_NAME}-server-0"
+  docker network disconnect "${NETWORK_ID}" "${CURRENT_CONTAINER}"
+  docker network rm -f "k3d-${CLUSTER_NAME}"
+}
+trap cleanup EXIT
+
+function pull_image() {
+  EXTERNAL_IMAGE=$1
+  echo "> Pull and import ${EXTERNAL_IMAGE} into cluster"
+  docker pull "${EXTERNAL_IMAGE}"
+  k3d image import "${EXTERNAL_IMAGE}" -c "${CLUSTER_NAME}"
+}
+
+function dump_logs() {
+  kubectl get pods -n cis-operator-system --show-labels
+  echo "RUNNER LOGS:"
+  kubectl logs -n cis-operator-system -l app.kubernetes.io/instance=security-scan-runner-k3s-e2e-scan || true
+  echo "SONOBUOY LOGS (rancher-kube-bench):"
+  kubectl logs -n cis-operator-system -l component=sonobuoy -c rancher-kube-bench || true
+  echo "SONOBUOY LOGS (sonobuoy-worker):"
+  kubectl logs -n cis-operator-system -l component=sonobuoy -c sonobuoy-worker || true    
+}
+
+cd $(dirname $0)/..
+
+echo "Running E2E tests"
+sleep "${E2E_TIMEOUT_SECONDS}" && cleanup | false &
+
+docker build -t local-k3s -f tests/Dockerfile.k3s tests
+
+echo "> Spinning up k3d cluster"
+# After a few executions k3d can have problems with evictions:
+# https://k3d.io/v5.0.1/faq/faq/#pods-evicted-due-to-lack-of-disk-space
+k3d cluster create "${CLUSTER_NAME}" --no-lb --kubeconfig-update-default --image local-k3s \
+  --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@server:0' \
+  --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@server:0'
+
+# Build image and import it into k3d.
+echo "> Build and load ${IMAGE} into cluster"
+docker build --build-arg ARCH -f package/Dockerfile -t "${IMAGE}" .
+k3d image import "${IMAGE}" -c "${CLUSTER_NAME}"
+
+pull_image "${SECURITY_SCAN_IMAGE}"
+pull_image "${SONOBUOY_IMAGE}"
+pull_image "${COREDNS_IMAGE}"
+pull_image "${HELM_IMAGE}"
+
+# Dapper will run on an isolated docker network.
+# To access k3d, grab the current container and connect it to k3d's network.
+NETWORK_ID=$(docker network ls -f name="k3d-${CLUSTER_NAME}" -q)
+docker network connect "${NETWORK_ID}" "${CURRENT_CONTAINER}"
+SERVER_IP=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{println .IPAddress}}{{end}}' "k3d-${CLUSTER_NAME}-server-0" | head -n1)
+
+# k3d's kubeconfig must be updated to the actual container IP.
+echo "> Update server to ${SERVER_IP}"
+kubectl config set-cluster "k3d-${CLUSTER_NAME}" --server="https://${SERVER_IP}:6443"
+
+# cis-operator may have intermittent issues if key components
+# from the cluster aren't ready.
+echo "> Wait for k3d base components to be ready"
+kubectl wait node "k3d-${CLUSTER_NAME}-server-0" --for=condition=ready --timeout=45s
+kubectl wait --timeout=60s --for=condition=ready -n kube-system pod -l app=local-path-provisioner
+kubectl wait --timeout=60s --for=condition=ready -n kube-system pod -l k8s-app=kube-dns
+
+echo "> Deploying cis-operator"
+kubectl apply -f ./crds
+kubectl apply -f ./tests/deploy.yaml
+
+echo "> Wait for cis-operator to be ready"
+# Can't kubectl wait before the deployment schedules the pod, so
+# wait 10 seconds for that to happen first.
+sleep 10s
+kubectl wait --for=condition=ready -n cis-operator-system pod -l cis.cattle.io/operator=cis-operator --timeout=30s
+
+echo "> Create ClusterScan"
+kubectl apply -f tests/k3s-bench-test.yaml
+
+docker exec "k3d-${CLUSTER_NAME}-server-0" /usr/local/bin/kube-apiserver &
+
+# Keep trying to check if the ClusterScan had any some tests that passed
+# that is good enough indication that all the mechanics of cis-operator
+# are working as expected.
+#
+# As soon as passing tests are detected, exit the e2e. If none is found,
+# the tests will eventually timeout based on E2E_TIMEOUT_SECONDS.
+while (true)
+do
+  if [ -n "${CANCELLING}" ]; then
+    break
+  fi
+
+  json=$(kubectl get ClusterScan k3s-e2e-scan -o jsonpath='{.status.summary}')
+  if [ -n "${json}" ]; then
+    passed=$(echo "${json}" | jq '.pass')
+    total=$(echo "${json}" | jq '.total')
+    fail=$(echo "${json}" | jq '.fail')
+    
+    if [ "${passed}" -gt "0" ]; then
+      echo "> cis-operator worked successfully"
+      
+      kubectl get ClusterScan -o yaml
+      echo "${json}" | jq .
+
+      exit 0
+    fi
+
+    if [ "${total}" == "${fail}" ]; then
+      echo "ERR: ALL TESTS FAILED!"
+      exit 1
+    fi
+  fi
+
+  dump_logs
+  sleep 2s
+done

tests/Dockerfile.k3s

@@ -0,0 +1,14 @@
+# This image is solely used for testing purposes
+# and aims to wrap around k3s, making any needed
+# changes for the cis-operator tests to work.
+FROM rancher/k3s:v1.25.6-k3s1
+
+# Upstream does not have files /etc/passwd nor /etc/group
+# which causes cis-operator to fail when scheduling a
+# running that maps those files from the "host".
+RUN echo "root:!:0:0::/:/bin/false" > /etc/passwd && \
+    touch /etc/group
+
+# A fake apiserver to trigger the if condition within
+# security-scan that runs kube-bench for the api-server.
+COPY kube-apiserver /usr/local/bin/kube-apiserver