Strict patch

Author: ktsakalozos

.github/workflows/build-snap.yml

@@ -1,12 +1,17 @@
 name: Build MicroK8s snap on PR and push to master
 
 on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+  - push
+  - pull_request
+
+### While we work on the strict feature we want the tests to run even if we do put PRs against the master.
+### When this work get merged into master the following should be commented in.
+#  push:
+#    branches:
+#      - master
+#  pull_request:
+#    branches:
+#      - master
 
 jobs:
   build:
@@ -43,24 +48,80 @@ jobs:
       - name: Running upgrade path test
         run: |
           set -x
-          sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade-path.py
-          sudo snap remove microk8s --purge
-      - name: Running addons tests
+          # Remove the snapd refresh as soon as v2.52 lands
+          sudo snap refresh snapd --channel=latest/edge
+      - name: Check branches
+        run: |
+          set -x
+          (cd tests; pytest -s verify-branches.py)
+      - name: Running addons tests in strict mode
         run: |
           set -x
-          sudo snap install *.snap --classic --dangerous
+          sudo snap install microk8s.snap --dangerous
+          sudo ./tests/connect-all-interfaces.sh
           ./tests/smoke-test.sh
           export UNDER_TIME_PRESSURE="True"
+          export SKIP_OPENEBS="True"
           export SKIP_PROMETHEUS="False"
           (cd tests; pytest -s verify-branches.py)
           sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py"
           sudo microk8s enable community
           sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/tests; pytest -s -ra test-addons.py"
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-strict-${{ strategy.job-index }}.tar.gz
           sudo snap remove microk8s --purge
-      - name: Running upgrade tests
+          sudo rm -rf $HOME/.kube
+          sudo rm -rf $HOME/.config/helm
+          sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log
+      - name: Upload strict inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-strict-actions
+          path: ./inspection-report-strict-${{ strategy.job-index }}.tar.gz
+      - name: Upload AppArmor denials
+        uses: actions/upload-artifact@v2
+        with:
+          name: apparmor-denials
+          path: ./denials-${{ strategy.job-index }}.log
+      - name: Running addons tests in devmode
         run: |
           set -x
-          sudo snap install *.snap --classic --dangerous
+          ################ Until devmode of docker-support is fixed we skip this part of the tests #######
+          exit 0
+          sudo snap install microk8s.snap --devmode --dangerous
+          sudo ./tests/connect-all-interfaces.sh
+          ./tests/smoke-test.sh
           export UNDER_TIME_PRESSURE="True"
-          sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/ ; UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade.py"
+          export SKIP_OPENEBS="False"
+          export SKIP_PROMETHEUS="False"
+          (cd tests; sudo -E pytest -s -ra test-addons.py)
+          sudo microk8s inspect |
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-devmode-${{ strategy.job-index }}.tar.gz
           sudo snap remove microk8s --purge
+      - name: Upload devmode inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-devmode-actions
+          path: ./inspection-report-devmode-${{ strategy.job-index }}.tar.gz
+      - name: Generate AppArmor on failure
+        run: sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log
+        if: failure()
+      - name: Upload AppArmor denials failure
+        uses: actions/upload-artifact@v2
+        with:
+          name: apparmor-denials
+          path: ./denials-${{ strategy.job-index }}.log
+        if: failure()
+      - name: Generate inspect tarball
+        run: >
+          sudo microk8s inspect |
+          grep -Po "Report tarball is at \K.+" |
+          sudo xargs -I {} mv {} inspection-report-fail-${{ strategy.job-index }}.tar.gz
+        if: failure()
+      - name: Upload inspect tarball
+        uses: actions/upload-artifact@v2
+        with:
+          name: inspection-report-actions
+          path: ./inspection-report-fail-${{ strategy.job-index }}.tar.gz
+        if: failure()

docs/build.md

@@ -83,9 +83,16 @@ lxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap .
 After copying it, you can install it with:
 
 ```shell
-snap install microk8s_*_amd64.snap --classic --dangerous
+sudo snap install microk8s_latest_amd64.snap --dangerous
 ```
 
+Finally, you need to connect the interfaces. To this end you can use the `connect-all-interfaces.sh` under the `tests` directory:
+
+```shell
+sudo tests/connect-all-interfaces.sh
+```
+
+
 ## Assembling the Calico CNI manifest
 
 The calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`.

microk8s-resources/default-args/kubelet

@@ -3,6 +3,7 @@
 --client-ca-file=${SNAP_DATA}/certs/ca.crt
 --anonymous-auth=false
 --root-dir=${SNAP_COMMON}/var/lib/kubelet
+--log-dir=${SNAP_COMMON}/var/log
 --fail-swap-on=false
 --feature-gates=DevicePlugins=true
 --eviction-hard="memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi"

microk8s-resources/wrappers/apiservice-kicker

@@ -59,10 +59,10 @@ do
     # every 5 seconds
     sleep 5
     if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] &&
-      getent group microk8s >/dev/null 2>&1
+      getent group snap_microk8s >/dev/null 2>&1
     then
       chmod -R ug+rwX ${SNAP_DATA}/var/kubernetes/backend || true
-      chgrp microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true
+      chgrp snap_microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true
     fi
 
     if ! [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ] &&
@@ -80,9 +80,8 @@ do
             echo "CSR change detected. Reconfiguring the kube-apiserver"
             rm -rf .srl
             snapctl stop microk8s.daemon-kubelite
-            snapctl stop microk8s.daemon-containerd
-            kill_all_container_shims
-            snapctl start microk8s.daemon-containerd
+            remove_all_containers
+            snapctl restart microk8s.daemon-containerd
             snapctl start microk8s.daemon-kubelite
             start_all_containers
             restart_attempt=$[$restart_attempt+1]

microk8s-resources/wrappers/microk8s-dbctl.wrapper

@@ -9,6 +9,8 @@ export PYTHONNOUSERSITE=false
 
 source $SNAP/actions/common/utils.sh
 
+exit_if_not_root
+
 exit_if_no_permissions
 
 LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py $@

microk8s-resources/wrappers/microk8s-disable.wrapper

@@ -6,9 +6,16 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
 ARCH="$($SNAP/bin/uname -m)"
 export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu"
 export PYTHONNOUSERSITE=false
+export LC_ALL="${LC_ALL:-C.UTF-8}"
+export LANG="${LANG:-C.UTF-8}"
+
+# avoid AppArmor denial in strict mode when running under sudo without -H
+cd "$SNAP"
 
 source $SNAP/actions/common/utils.sh
 
+exit_if_not_root
+
 exit_if_no_permissions
 
 LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/disable.py $@

microk8s-resources/wrappers/microk8s-enable.wrapper

@@ -6,9 +6,16 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
 ARCH="$($SNAP/bin/uname -m)"
 export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu"
 export PYTHONNOUSERSITE=false
+export LC_ALL="${LC_ALL:-C.UTF-8}"
+export LANG="${LANG:-C.UTF-8}"
+
+# avoid AppArmor denial in strict mode when running under sudo without -H
+cd "$SNAP"
 
 source $SNAP/actions/common/utils.sh
 
+exit_if_not_root
+
 exit_if_no_permissions
 
 LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py $@

microk8s-resources/wrappers/microk8s-join.wrapper

@@ -22,4 +22,4 @@ fi
 
 exit_if_no_permissions
 
-sudo -E  LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/join.py $@
+LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/join.py $@

microk8s-resources/wrappers/microk8s-kubectl.wrapper

@@ -30,7 +30,7 @@ fi
 declare -a args="($(cat $SNAP_DATA/args/kubectl))"
 if [ -n "${args[@]-}" ]
 then
-  "${SNAP}/kubectl" "${args[@]}" "$@"
+  EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "${args[@]}" "$@"
 else
-  "${SNAP}/kubectl" "$@"
+  EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "$@"
 fi

microk8s-resources/wrappers/microk8s-leave.wrapper

@@ -11,6 +11,7 @@ export PYTHONNOUSERSITE=false
 source $SNAP/actions/common/utils.sh
 
 exit_if_stopped
+exit_if_not_root
 exit_if_no_permissions
 
 if ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ] &&
@@ -20,4 +21,4 @@ then
   exit 1
 fi
 
-run_with_sudo preserve_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@
+run_with_sudo ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@