.github: Add k8s conformance tests in Nightly

TeddyAndrieux · TeddyAndrieux · commit b47c656d90c3 · 2023-02-03T12:58:10.000+01:00
diff --git a/.github/actions/run-k8s-conformance/action.yaml b/.github/actions/run-k8s-conformance/action.yaml
@@ -0,0 +1,77 @@
+name: "Run K8s Conformance"
+description: "Run the Kubernetes Conformance tests"
+
+inputs:
+  metalk8s-short-version:
+    description: "The MetalK8s short version to use in the PR content"
+    required: true
+  dest:
+    description: "Destination directory for results + PR content"
+    required: false
+    default: "artifacts/conformance"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create result directory
+      shell: bash
+      run: |
+        result_dir="${{ inputs.dest }}/sonobuoy-results"
+        echo "result_dir=$result_dir" >> $GITHUB_ENV
+        mkdir -p "$result_dir"
+    - name: Retrieve the Kubernetes version
+      id: get-k8s-version
+      uses: ./.github/actions/run-command-ssh
+      with:
+        COMMAND: |
+          sudo rpm -q --queryformat '%{VERSION}' kubelet | cut -d'.' -f1,2
+        CAPTURE_RESULT: "true"
+    - name: Get sonobuoy bin
+      shell: bash
+      env:
+        SONOBUOY_VERSION: "0.56.15"
+      run: |
+        curl -Lo "sonobuoy.tar.gz" https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz
+        tar xvf sonobuoy.tar.gz
+    - name: Copy sonobuoy bin to Bootstrap node
+      uses: ./.github/actions/copy-file-ssh
+      with:
+        SOURCE_FILE: "sonobuoy"
+    - name: Run conformance tests from Bootstrap node
+      uses: ./.github/actions/run-command-ssh
+      with:
+        COMMAND: |
+          sudo ./sonobuoy run --kubeconfig=/etc/kubernetes/admin.conf --mode=certified-conformance --wait
+        SSH_OPTIONS: "-o ServerAliveInterval=15"
+    - name: Retrieve conformance tests result
+      uses: ./.github/actions/run-command-ssh
+      with:
+        COMMAND: |
+          sudo ./sonobuoy retrieve --kubeconfig=/etc/kubernetes/admin.conf --filename sonobuoy_result.tar.gz
+    - name: Retrieve conformance tests result from Bootstrap node
+      uses: ./.github/actions/copy-file-ssh
+      with:
+        NODE_TO:  ""
+        NODE_FROM: "bootstrap"
+        SOURCE_FILE: "sonobuoy_result.tar.gz"
+        DESTINATION_FILE: "${{ env.result_dir }}/"
+    - name: Extract conformance tests result
+      shell: bash
+      working-directory: ${{ env.result_dir }}
+      run: tar xvf sonobuoy_result.tar.gz
+    - name: Check conformance tests result
+      shell: bash
+      run: |
+        failed_tests=$(./sonobuoy results "$result_dir/sonobuoy_result.tar.gz" --mode=detailed --plugin=e2e | jq 'select(.status=="failed")')
+        [ -n "$failed_tests" ] && echo $failed_tests && exit 1 || exit 0
+    - name: Prepare conformance PR content
+      shell: bash
+      env:
+        DIRECTORY: "${{ inputs.dest }}/pr-content"
+        SONOBUOY_RES_DIR: "${{ env.result_dir }}"
+        K8S_VERSION: "${{ steps.get-k8s-version.outputs.RESULT }}"
+        METALK8S_VERSION: "${{ inputs.metalk8s-short-version }}"
+      run: |
+        .github/scripts/build-conformance-pr-content.sh
+        tar cvf "${{ inputs.dest }}/pr-content.tar.gz" -C "$DIRECTORY" "v$K8S_VERSION"
+
diff --git a/.github/scripts/build-conformance-pr-content.sh b/.github/scripts/build-conformance-pr-content.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+
+DIRECTORY=${DIRECTORY:-pr-content}
+SONOBUOY_RES_DIR=${SONOBUOY_RES_DIR:-sonobuoy-results}
+
+K8S_VERSION=${K8S_VERSION:-}
+METALK8S_VERSION=${METALK8S_VERSION:-}
+
+dest="${DIRECTORY}/v${K8S_VERSION}/MetalK8s"
+doc_url="https://metal-k8s.readthedocs.io/en/development-${METALK8S_VERSION}/"
+
+mkdir -p "$dest"
+
+cat > "$dest/PRODUCT.yaml" << EOF
+vendor: Scality
+name: MetalK8s
+description: "An opinionated Kubernetes distribution with a focus on long-term on-prem deployments"
+version: ${METALK8S_VERSION}
+type: distribution
+website_url: https://github.com/scality/metalk8s/
+repo_url: https://github.com/scality/metalk8s.git
+product_logo_url: https://raw.githubusercontent.com/scality/metalk8s/development/${METALK8S_VERSION}/artwork/metalk8s-logo-vertical.svg
+documentation_url: ${doc_url}
+EOF
+
+sed "s%@@DOC_URL@@%${doc_url}%g" > "$dest/README.md" << 'ENDREADME'
+# MetalK8s
+Official documentation: @@DOC_URL@@
+
+## Prerequisites
+- An OpenStack cluster
+- The official CentOS 7.9 2009 image pre-loaded in Glance
+- Three VMs with 8 vCPUs, 16 GB of RAM, 40GB of local storage
+
+## Provisioning
+- Create two private network in the OpenStack cluster with port security
+  disabled, and a subnet in each:
+
+  * Control-plane network: 192.168.1.0/24
+  * Workload-plane network: 192.168.2.0/24
+
+- Create VM instances using the CentOS 7.9 image, and attach each of them to a
+  public network (for internet access) and the two private networks.
+
+- Configure the interface for private networks (make sure to fill in the
+  appropriate MAC address):
+
+  ```
+  $ cat > /etc/sysconfig/network-scripts/ifcfg-eth1 << EOF
+  BOOTPROTO=dhcp
+  DEVICE=eth1
+  HWADDR=...
+  ONBOOT=yes
+  TYPE=Ethernet
+  USERCTL=no
+  PEERDNS=no
+  EOF
+  $ cat > /etc/sysconfig/network-scripts/ifcfg-eth2 << EOF
+  BOOTPROTO=dhcp
+  DEVICE=eth2
+  HWADDR=...
+  ONBOOT=yes
+  TYPE=Ethernet
+  USERCTL=no
+  PEERDNS=no
+  EOF
+  $ systemctl restart network
+  ```
+
+### Provisioning the Bootstrap Node
+On one of the VMs, which will act as the *bootstrap* node, perform the following
+steps:
+
+- Set up the Salt Minion ID:
+
+  ```
+  $ mkdir /etc/salt; chmod 0700 /etc/salt
+  $ echo metalk8s-bootstrap > /etc/salt/minion_id
+  ```
+
+- Download MetalK8s ISO to `/home/centos/metalk8s.iso`
+
+- Create `/etc/metalk8s/bootstrap.yaml`:
+
+  ```
+  $ mkdir /etc/metalk8s
+  $ cat > /etc/metalk8s/bootstrap.yaml << EOF
+  apiVersion:  metalk8s.scality.com/v1alpha3
+  kind: BootstrapConfiguration
+  networks:
+    controlPlane:
+        cidr: 192.168.1.0/24
+    workloadPlane:
+        cidr: 192.168.2.0/24
+    portmap:
+        cidr: 0.0.0.0/0
+    nodeport:
+        cidr: 0.0.0.0/0
+  ca:
+    minion: metalk8s-bootstrap
+  archives:
+    - /home/centos/metalk8s.iso
+  EOF
+  ```
+
+- Bootstrap the cluster
+
+  ```
+  $ mkdir /mnt/metalk8s
+  $ mount /home/centos/metalk8s.iso /mnt/metalk8s
+  $ cd /mnt/metalk8s
+  $ ./bootstrap.sh
+  ```
+
+### Provisioning the Cluster Nodes
+Add the 2 other nodes to the cluster according to the procedure outlined in the
+MetalK8s documentation. The easiest way to achieve this is through the MetalK8s
+UI.
+
+## Preparing the Cluster to Run Sonobuoy
+On the *bootstrap* node:
+
+- Configure access to the Kubernetes API server
+
+  ```
+  $ export KUBECONFIG=/etc/kubernetes/admin.conf
+  ```
+
+- Remove taints from the node, which would prevent the Sonobuoy *Pod*s from
+  being scheduled:
+
+  ```
+  $ kubectl taint node metalk8s-bootstrap node-role.kubernetes.io/bootstrap-
+  node/metalk8s-bootstrap untainted
+  $ kubectl taint node metalk8s-bootstrap node-role.kubernetes.io/infra-
+  node/metalk8s-bootstrap untainted
+  ```
+
+## Running Sonobuoy and Collecting Results
+Follow the
+[instructions](https://github.com/cncf/k8s-conformance/blob/master/instructions.md)
+as found in the [CNCF K8s Conformance repository](https://github.com/cncf/k8s-conformance).
+ENDREADME
+
+cp "${SONOBUOY_RES_DIR}/plugins/e2e/results/global/e2e.log" "$dest/"
+cp "${SONOBUOY_RES_DIR}/plugins/e2e/results/global/junit_01.xml" "$dest/"
diff --git a/.github/workflows/multi-node-test.yaml b/.github/workflows/multi-node-test.yaml
@@ -41,6 +41,11 @@ on:
         required: false
         type: boolean
         default: false
+      k8s-conformance:
+        description: "Whether or not k8s conformance tests should be ran"
+        required: false
+        type: boolean
+        default: false
       enable-debug-when-failure:
         description: "Whether or not debug when failure should be enabled or not"
         required: false
@@ -82,7 +87,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Install deps
-        run: sudo yum install -y isomd5sum
+        run: sudo yum install -y isomd5sum jq
 
      ## Spawn {{{
       - name: Export environment variables for accessing the cloud
@@ -262,15 +267,17 @@ jobs:
           SOURCE_FILE: "metalk8s.iso"
       - name: Mount MetalK8s ISO on the new Bootstrap node
         if: inputs.bootstrap-restore
-        id: get-new-mnt
+        id: mount-iso
         uses: ./.github/actions/mount-iso
       - name: Run restore tests from Bastion
         if: inputs.bootstrap-restore
         uses: ./.github/actions/bastion-tests
         with:
           PYTEST_FILTERS: "restore"
           BOOTSTRAP_BACKUP_ARCHIVE: "/tmp/${{ steps.backup.outputs.filename }}"
-          mountpoint: ${{ steps.get-new-mnt.outputs.mountpoint }}
+          # NOTE: We need to specify the mountpoint since it will not be
+          # able to auto-discover it because the bootstrap node is down
+          mountpoint: ${{ steps.mount-iso.outputs.mountpoint }}
       - name: Wait for pods to stabilize after restore
         if: inputs.bootstrap-restore
         uses: ./.github/actions/wait-pod-stable
@@ -281,6 +288,42 @@ jobs:
           PYTEST_FILTERS: "post and ci and not slow"
       # }}}
 
+      # Retrieve information {{{
+      - name: Retrieve product.txt from artifacts
+        run: >
+          curl --fail -LO -u ${{ secrets.ARTIFACTS_USER }}:${{ secrets.ARTIFACTS_PASSWORD }}
+          ${{ inputs.artifacts-url }}/product.txt
+      - name: Get full MetalK8s version
+        run: |
+          source product.txt
+          echo "METALK8S_VERSION=$VERSION" >> $GITHUB_ENV
+          echo "METALK8S_SHORT_VERSION=$SHORT_VERSION" >> $GITHUB_ENV
+      # }}}
+
+      # Run k8s conformance tests (if enabled) {{{
+      - name: Untaint bootstrap node for conformance tests
+        if: inputs.k8s-conformance
+        uses: ./.github/actions/untaint
+      - name: Expose nodeport and portmap on every IPs
+        if: inputs.k8s-conformance
+        uses: ./.github/actions/run-command-ssh
+        with:
+          COMMAND: |
+            sudo salt-call --local --retcode-passthrough state.single file.serialize /etc/metalk8s/bootstrap.yaml dataset='{"networks": {"nodeport": {"cidr": "0.0.0.0/0"}, "portmap": {"cidr": "0.0.0.0/0"}}}' merge_if_exists=True
+            SALT_MASTER=\$(sudo crictl ps --label="io.kubernetes.container.name=salt-master" -q)
+            sudo crictl exec \$SALT_MASTER salt-run state.sls metalk8s.kubernetes.kube-proxy.deployed saltenv=metalk8s-$METALK8S_VERSION
+            sudo crictl exec \$SALT_MASTER salt-run state.sls metalk8s.kubernetes.cni.calico.deployed saltenv=metalk8s-$METALK8S_VERSION
+            sudo crictl exec \$SALT_MASTER salt '*' state.sls metalk8s.addons.nginx-ingress.certs saltenv=metalk8s-$METALK8S_VERSION
+      - name: Wait for pods to stabilize before conformance tests
+        if: inputs.k8s-conformance
+        uses: ./.github/actions/wait-pod-stable
+      - name: Run the k8s conformance tests
+        if: inputs.k8s-conformance
+        uses: ./.github/actions/run-k8s-conformance
+        with:
+          metalk8s-short-version: "${{ env.METALK8S_SHORT_VERSION }}"
+      # }}}
+
       - name: Generate and Collect sosreport
         if: always()
         uses: ./.github/actions/sosreport-logs
@@ -299,16 +342,9 @@ jobs:
       ## }}
 
       # Generate snapshots (if enabled) {{{
-      - name: Retrieve product.txt from artifacts
-        if: inputs.generate-snapshots
-        run: >
-          curl --fail -LO -u ${{ secrets.ARTIFACTS_USER }}:${{ secrets.ARTIFACTS_PASSWORD }}
-          ${{ inputs.artifacts-url }}/product.txt
-      - name: Get full MetalK8s version and total number of nodes
+      - name: Get total number of nodes
         if: inputs.generate-snapshots
         run: |
-          source product.txt
-          echo "METALK8S_VERSION=$VERSION" >> $GITHUB_ENV
           echo "TOTAL_NB_NODES=$((${{ env.NODES_COUNT }} + 1))" >> $GITHUB_ENV
       - name: Generate snapshot
         if: inputs.generate-snapshots
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -190,13 +190,25 @@ jobs:
       nodes-count: 2
       bootstrap-restore: true
 
+  k8s-conformance:
+    uses: ./.github/workflows/multi-node-test.yaml
+    secrets: inherit
+    needs:
+      - retrieve-info
+    with:
+      name: k8s-conformance
+      artifacts-url: ${{ needs.retrieve-info.outputs.artifacts-link }}
+      nodes-count: 2
+      k8s-conformance: true
+
   write-final-failed-status:
     runs-on: ubuntu-22.04
     needs:
       - lifecycle-promoted
       - lifecycle-dev
       - install
       - bootstrap-restore
+      - k8s-conformance
     if: failure()
     steps:
       - name: Checkout
@@ -223,6 +235,7 @@ jobs:
       - lifecycle-dev
       - install
       - bootstrap-restore
+      - k8s-conformance
     if: success()
     steps:
       - name: Checkout
