Cleanup Old E2E Namespaces #3516
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Cleanup Old E2E Namespaces" | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 * * * *" # every hour; adjust as needed | |
| jobs: | |
| discover_cleanup: | |
| name: List namespaces to cleanup | |
| runs-on: self-hosted | |
| outputs: | |
| namespaces: ${{ steps.discover_ns.outputs.namespaces }} | |
| steps: | |
| - name: Cleanup old namespaces except keep_namespace=true | |
| id: discover_ns | |
| env: | |
| # Set how old namespaces must be to be purged (in seconds) | |
| AGE_THRESHOLD: 10800 # Default 3 hours | |
| run: | | |
| THRESHOLD=${AGE_THRESHOLD} | |
| NOW=$(date +%s) | |
| echo "⚠️ Deleting namespaces older than $((THRESHOLD/3600)) hours unless keep_namespace=true:" | |
| NAMESPACES="$(kubectl get ns -o json | jq -r '.items[] | select(.metadata.labels.keep_namespace != "true") | .metadata.name' | grep opencrvs | grep -v 'opencrvs-deps-e2e')" || NAMESPACES="" | |
| KEEP_NAMESPACES=$(kubectl get ns -o json | jq -r '.items[] | select(.metadata.labels.keep_namespace == "true") | .metadata.name') | |
| NAMESPACES_TO_CLEANUP=() | |
| for ns in $NAMESPACES; do | |
| UPDATED_AT=$(kubectl get ns "$ns" -o jsonpath='{.metadata.labels.updated_at}') | |
| AGE=$((NOW - UPDATED_AT)) | |
| if [ -z "$UPDATED_AT" ]; then | |
| echo " - ℹ️ Skipping namespace $ns, no update_at label" | |
| elif [ "$AGE" -ge "$THRESHOLD" ]; then | |
| echo " - 🧹 Adding namespace $ns to cleanup list" | |
| NAMESPACES_TO_CLEANUP+=("$ns") | |
| else | |
| echo " - 🕒 Skipping namespace $ns (not old enough: $((AGE/3600))h $(( (AGE%3600)/60 ))m)" | |
| fi | |
| done | |
| namespaces_json=$(printf '%s\n' "${NAMESPACES_TO_CLEANUP[@]}" | jq -R -s -c 'split("\n")[:-1]') | |
| echo "namespaces=$namespaces_json" >> $GITHUB_OUTPUT | |
| echo "🔒 Following namespaces will be skipped, label keep_namespace=true:" | |
| printf '\t- %s\n' $KEEP_NAMESPACES | |
| echo "🗑️ Following namespaces will be removed:" | |
| printf '\t- %s\n' "${NAMESPACES_TO_CLEANUP[@]}" | |
| cleanup: | |
| needs: discover_cleanup | |
| runs-on: self-hosted | |
| if: ${{ needs.discover_cleanup.outputs.namespaces != '[""]' && needs.discover_cleanup.outputs.namespaces != '[]' && needs.discover_cleanup.outputs.namespaces != '' }} | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| namespace: ${{ fromJson(needs.discover_cleanup.outputs.namespaces) }} | |
| name: Remove ${{ matrix.namespace }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Create env variables | |
| id: env_vars | |
| run: | | |
| echo "ENV=$(echo ${{ matrix.namespace }} | sed 's/opencrvs-//')" >> $GITHUB_OUTPUT | |
| echo "CORE_IMAGE_TAG=$(helm get values opencrvs -n ${{ matrix.namespace }} -ojson | jq .image.tag)" >> $GITHUB_OUTPUT | |
| - name: Update k8s-env/opencrvs/values.yaml | |
| env: | |
| ENV: ${{ steps.env_vars.outputs.ENV }} | |
| run: sed -i -e "s#{{STACK}}#${ENV}#g" k8s-env/opencrvs/values.yaml | |
| - name: Cleanup environment | |
| env: | |
| ENV: ${{ steps.env_vars.outputs.ENV }} | |
| CORE_IMAGE_TAG: ${{ steps.env_vars.outputs.CORE_IMAGE_TAG }} | |
| run: | | |
| kubectl delete job -n ${{ matrix.namespace }} --ignore-not-found=true data-cleanup | |
| helm template -f k8s-env/opencrvs/values.yaml \ | |
| --set data_cleanup.enabled=true \ | |
| --set data_cleanup.env.make_bucket=false \ | |
| --namespace opencrvs-${ENV} \ | |
| -s templates/data-cleanup-job.yaml \ | |
| oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n opencrvs-${ENV} --wait=true -f - | |
| sleep 30; | |
| kubectl logs job/data-cleanup -f --all-containers=true -n opencrvs-${ENV} || true | |
| kubectl wait --for=condition=complete job/data-cleanup -n opencrvs-${ENV} --timeout=600s; | |
| - name: Uninstall helm release | |
| run: helm uninstall opencrvs -n ${{ matrix.namespace }} || echo "Helm release is not installed" | |
| - name: Delete namespace ${{ matrix.namespace }} | |
| run: kubectl delete ns ${{ matrix.namespace }} |