Support and document cleanup hooks

[blu3r4y]

When users uninstall Koney without removing deception policies first, the finalizers which would remove the traps never get executed, resulting in deception policies that cannot be deleted.

We should recommend users to run the following before uninstalling Koney:

kubectl delete deceptionpolicy --ignore-not-found --wait --all
kubectl delete deceptionalertsink --ignore-not-found --wait --all --all-namespaces 

Additionally, we could add a Helm hook that ensures that those custom resources a removed before uninstalling the operator itself. The following is a draft:

apiVersion: v1
kind: ServiceAccount
metadata:
  name: {{ .Release.Name }}-cleanup
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "cleanup.labels" . | nindent 4 }}
  annotations:
    {{- include "cleanup.annotations" . | nindent 4 }}
    helm.sh/hook-weight: "-5"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: {{ .Release.Name }}-cleanup
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "cleanup.labels" . | nindent 4 }}
  annotations:
    {{- include "cleanup.annotations" . | nindent 4 }}
    helm.sh/hook-weight: "-5"
rules:
- apiGroups: ["research.dynatrace.com"]
  resources: ["deceptionpolicies", "deceptionalertsinks"]
  verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: {{ .Release.Name }}-cleanup
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "cleanup.labels" . | nindent 4 }}
  annotations:
    {{- include "cleanup.annotations" . | nindent 4 }}
    helm.sh/hook-weight: "-5"
subjects:
- kind: ServiceAccount
  name: {{ .Release.Name }}-cleanup
  namespace: {{ .Release.Namespace }}
roleRef:
  kind: Role
  name: {{ .Release.Name }}-cleanup
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: Job
metadata:
  name: {{ .Release.Name }}-cleanup
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "cleanup.labels" . | nindent 4 }}
  annotations:
    {{- include "cleanup.annotations" . | nindent 4 }}
    helm.sh/hook-weight: "0"
spec:
  template:
    spec:
      serviceAccountName: {{ .Release.Name }}-cleanup
      containers:
      - name: kubectl
        image: cgr.dev/chainguard/kubectl:latest
        command:
        - /usr/bin/kubectl
        - delete
        - deceptionpolicy,deceptionalertsink
        - --namespace={{ .Release.Namespace }}
        - --ignore-not-found
        - --timeout=60s
        - --wait
        - --all
      restartPolicy: Never

# _helpers.tpl
{{/*
Common labels for cleanup resources
*/}}
{{- define "cleanup.labels" -}}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/part-of: {{ .Chart.Name }}
helm.sh/chart: {{ .Chart.Version | quote }}
{{- end }}

{{/*
Common annotations for cleanup resources
*/}}
{{- define "cleanup.annotations" -}}
helm.sh/hook: pre-delete
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
{{- end }}