k8s: add garbage collection script & cron job

roles/common/files/k8s/cleanup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Garbage collect completed (or very old) jobs
+#
+for ctx in $(kubectl config get-contexts --output=name); do
+    KUBECTL="kubectl --context=$ctx"
+
+    echo "# ===== $ctx ===== "
+    # cleanup successful jobs
+    $KUBECTL delete jobs --field-selector status.successful==1
+
+    # Cleanup failed pods
+    $KUBECTL delete pods --field-selector status.phase=Failed
+
+    # delete jobs older than 1d (have 'd' in the AGE field)
+    $KUBECTL delete job $($KUBECTL get job | awk 'match($4,/[0-9]+d/) {print $1}')
+done

roles/common/tasks/k8s.yml

@@ -231,3 +231,35 @@
       kubectl --context $ctx create secret generic {{ item.name }} --from-literal=token={{ item.token }} --dry-run=client -o yaml | kubectl --context $ctx apply -f -
     done
   loop: "{{ kci_tokens.tokens }}"
+
+- name: Create /home/buildslave/bin
+  tags:
+    - k8s-scripts
+    - never
+  file:
+    path: "/home/buildslave/bin"
+    state: directory
+    owner: buildslave
+    group: buildslave
+    mode: 0755
+
+- name: Add k8s helper shell scripts
+  tags:
+    - k8s-scripts
+    - never
+  copy:
+    src: k8s/cleanup.sh
+    dest: "/home/buildslave/bin/k8s-cleanup.sh"
+    owner: buildslave
+    group: buildslave
+    mode: 0755
+
+- name: Daily cron for k8s garbage collection
+  tags:
+    - k8s-scripts
+    - never
+  ansible.builtin.cron:
+    name: "k8s garbage collection"
+    special_time: "daily"
+    job: "/home/buildslave/bin/k8s-cleanup.sh"
+    user: buildslave