|
| 1 | +// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. |
| 2 | +// SPDX-License-Identifier: Apache-2.0 |
| 3 | +import * as k8s from '@pulumi/kubernetes'; |
| 4 | +import * as pulumi from '@pulumi/pulumi'; |
| 5 | +import { versionFromDefault } from '@lfdecentralizedtrust/splice-pulumi-common/src/version'; |
| 6 | + |
| 7 | +import { DOCKER_REPO, infraAffinityAndTolerations } from '../../common'; |
| 8 | + |
| 9 | +const cronJobName = 'gc-pod-reaper-job'; |
| 10 | +const reaperNamespace = 'gc-pod-reaper'; |
| 11 | +const serviceAccountName = 'gc-pod-reaper-service-account'; |
| 12 | +const schedule = '0 * * * *'; // Run once every hour, at minute 0 |
| 13 | + |
| 14 | +const deleteBadPodsCommand = [ |
| 15 | + '/bin/bash', |
| 16 | + '-c', |
| 17 | + ` |
| 18 | + echo "--- $(date) Starting Pod Reaper ---"; |
| 19 | + TARGET_NAMESPACES_LIST=$(echo "$TARGET_NAMESPACES" | tr ',' ' '); |
| 20 | + echo "Targeting namespaces: $TARGET_NAMESPACES_LIST"; |
| 21 | + if [ -z "$TARGET_NAMESPACES_LIST" ]; then |
| 22 | + echo "Error: No target namespaces provided. Exiting."; |
| 23 | + exit 1 |
| 24 | + fi |
| 25 | + echo "--- Starting Cleanup Loop ---"; |
| 26 | + for NAMESPACE in $TARGET_NAMESPACES_LIST; do |
| 27 | + echo "Processing namespace: $NAMESPACE"; |
| 28 | + echo "Listing all pods in $NAMESPACE:"; |
| 29 | + kubectl get pods -n "$NAMESPACE"; |
| 30 | + echo "--- checking PODS with bad status in $NAMESPACE ---"; |
| 31 | + BAD_PODS=$( |
| 32 | + kubectl get pods -n "$NAMESPACE" -o json | \\ |
| 33 | + jq -r '.items[] | |
| 34 | + (select(.status.phase == "Unknown" and .status.reason == "ContainerStatusUnknown") | |
| 35 | + .metadata.name) // |
| 36 | + (select(.status.containerStatuses[]?.state.terminated? | |
| 37 | + .reason == "Error" and .exitCode == 137) | |
| 38 | + .metadata.name)' |
| 39 | + ); |
| 40 | + if [ -z "$BAD_PODS" ]; then |
| 41 | + echo "No bad pods found in $NAMESPACE. Skipping."; |
| 42 | + continue |
| 43 | + fi |
| 44 | + echo "Found bad pods in $NAMESPACE: $BAD_PODS"; |
| 45 | + for POD_NAME in $BAD_PODS; do |
| 46 | + echo "Attempting to delete pod $NAMESPACE/$POD_NAME"; |
| 47 | + kubectl delete pod -n "$NAMESPACE" "$POD_NAME" |
| 48 | + if [ $? -eq 0 ]; then |
| 49 | + echo "Successfully deleted $NAMESPACE/$POD_NAME"; |
| 50 | + else |
| 51 | + echo "Failed to delete $NAMESPACE/$POD_NAME"; |
| 52 | + fi |
| 53 | + done |
| 54 | + echo "Finished cleanup for $NAMESPACE."; |
| 55 | + echo "---" |
| 56 | + done |
| 57 | + echo "--- $(date) Pod Reaper Finished ---"; |
| 58 | + true |
| 59 | + `, |
| 60 | +]; |
| 61 | + |
| 62 | +export function deployGCPodReaper( |
| 63 | + name: string, |
| 64 | + targetNamespaces: string[], |
| 65 | + opts?: pulumi.ComponentResourceOptions |
| 66 | +): k8s.batch.v1.CronJob { |
| 67 | + const ns = new k8s.core.v1.Namespace(name, { |
| 68 | + metadata: { |
| 69 | + name: reaperNamespace, |
| 70 | + labels: { |
| 71 | + 'app.kubernetes.io/name': reaperNamespace, |
| 72 | + }, |
| 73 | + }, |
| 74 | + }); |
| 75 | + |
| 76 | + targetNamespaces.forEach(namespace => { |
| 77 | + const podManagementRole = new k8s.rbac.v1.Role( |
| 78 | + namespace + '-gc-pod-reaper-role', |
| 79 | + { |
| 80 | + metadata: { |
| 81 | + name: namespace + '-gc-pod-reaper-role', |
| 82 | + namespace: namespace, |
| 83 | + }, |
| 84 | + rules: [ |
| 85 | + { |
| 86 | + apiGroups: [''], // Core API group for Pods |
| 87 | + resources: ['pods'], |
| 88 | + verbs: ['list', 'create', 'delete', 'update'], |
| 89 | + }, |
| 90 | + ], |
| 91 | + }, |
| 92 | + { parent: ns } |
| 93 | + ); |
| 94 | + |
| 95 | + new k8s.rbac.v1.RoleBinding( |
| 96 | + namespace + '-gc-pod-reaper-pod-manager-binding', |
| 97 | + { |
| 98 | + metadata: { |
| 99 | + name: namespace + 'gc-pod-reaper-pod-manager-binding', |
| 100 | + namespace: namespace, |
| 101 | + }, |
| 102 | + subjects: [ |
| 103 | + { |
| 104 | + kind: 'ServiceAccount', |
| 105 | + name: serviceAccountName, |
| 106 | + namespace: 'gc-pod-reaper', |
| 107 | + }, |
| 108 | + ], |
| 109 | + roleRef: { |
| 110 | + kind: 'Role', |
| 111 | + name: podManagementRole.metadata.name, |
| 112 | + apiGroup: 'rbac.authorization.k8s.io', |
| 113 | + }, |
| 114 | + }, |
| 115 | + { parent: podManagementRole, dependsOn: [podManagementRole] } |
| 116 | + ); |
| 117 | + }); |
| 118 | + |
| 119 | + const targetNamespacesEnv = targetNamespaces.join(','); |
| 120 | + |
| 121 | + new k8s.core.v1.ServiceAccount( |
| 122 | + serviceAccountName, |
| 123 | + { |
| 124 | + metadata: { |
| 125 | + name: serviceAccountName, |
| 126 | + namespace: ns.metadata.name, |
| 127 | + }, |
| 128 | + }, |
| 129 | + { parent: ns } |
| 130 | + ); |
| 131 | + |
| 132 | + return new k8s.batch.v1.CronJob( |
| 133 | + cronJobName, |
| 134 | + { |
| 135 | + metadata: { |
| 136 | + name: cronJobName, |
| 137 | + namespace: ns.metadata.name, |
| 138 | + }, |
| 139 | + spec: { |
| 140 | + schedule: schedule, |
| 141 | + concurrencyPolicy: 'Forbid', |
| 142 | + successfulJobsHistoryLimit: 2, |
| 143 | + failedJobsHistoryLimit: 2, |
| 144 | + jobTemplate: { |
| 145 | + metadata: { |
| 146 | + labels: { |
| 147 | + app: 'gc-pod-reaper', |
| 148 | + }, |
| 149 | + }, |
| 150 | + spec: { |
| 151 | + template: { |
| 152 | + spec: { |
| 153 | + serviceAccountName: serviceAccountName, |
| 154 | + restartPolicy: 'OnFailure', |
| 155 | + ...infraAffinityAndTolerations, |
| 156 | + containers: [ |
| 157 | + { |
| 158 | + name: cronJobName, |
| 159 | + image: `${DOCKER_REPO}/splice-debug:${versionFromDefault()}`, |
| 160 | + imagePullPolicy: 'Always', |
| 161 | + command: deleteBadPodsCommand, |
| 162 | + env: [ |
| 163 | + { |
| 164 | + name: 'TARGET_NAMESPACES', |
| 165 | + value: targetNamespacesEnv, |
| 166 | + }, |
| 167 | + ], |
| 168 | + }, |
| 169 | + ], |
| 170 | + }, |
| 171 | + }, |
| 172 | + }, |
| 173 | + }, |
| 174 | + }, |
| 175 | + }, |
| 176 | + { parent: ns, ...opts } |
| 177 | + ); |
| 178 | +} |
0 commit comments