|
| 1 | +apiVersion: v1 |
| 2 | +kind: ServiceAccount |
| 3 | +metadata: |
| 4 | + name: restore-controller |
| 5 | + namespace: longhorn-system |
| 6 | +--- |
| 7 | +apiVersion: rbac.authorization.k8s.io/v1 |
| 8 | +kind: ClusterRole |
| 9 | +metadata: |
| 10 | + name: restore-controller-role |
| 11 | +rules: |
| 12 | + - apiGroups: ["longhorn.io"] |
| 13 | + resources: ["backupvolumes", "backups"] |
| 14 | + verbs: ["get", "list", "watch"] |
| 15 | + - apiGroups: [""] |
| 16 | + resources: ["persistentvolumes", "persistentvolumeclaims"] |
| 17 | + verbs: ["get", "list", "create", "patch"] |
| 18 | + - apiGroups: [""] |
| 19 | + resources: ["events"] |
| 20 | + verbs: ["create", "patch"] |
| 21 | +--- |
| 22 | +apiVersion: rbac.authorization.k8s.io/v1 |
| 23 | +kind: ClusterRoleBinding |
| 24 | +metadata: |
| 25 | + name: restore-controller-binding |
| 26 | +subjects: |
| 27 | + - kind: ServiceAccount |
| 28 | + name: restore-controller |
| 29 | + namespace: longhorn-system |
| 30 | +roleRef: |
| 31 | + kind: ClusterRole |
| 32 | + name: restore-controller-role |
| 33 | + apiGroup: rbac.authorization.k8s.io |
| 34 | +--- |
| 35 | +apiVersion: batch/v1 |
| 36 | +kind: Job |
| 37 | +metadata: |
| 38 | + name: restore-critical-volumes |
| 39 | + namespace: longhorn-system |
| 40 | + annotations: |
| 41 | + "helm.sh/hook": post-install,post-upgrade |
| 42 | + "helm.sh/hook-weight": "5" |
| 43 | + "argocd.argoproj.io/sync-wave": "2" # Run after Longhorn (Wave 1) but before Apps |
| 44 | +spec: |
| 45 | + template: |
| 46 | + spec: |
| 47 | + serviceAccountName: restore-controller |
| 48 | + restartPolicy: OnFailure |
| 49 | + containers: |
| 50 | + - name: restore-controller |
| 51 | + image: bitnami/kubectl:latest |
| 52 | + command: ["/bin/bash", "-c"] |
| 53 | + args: |
| 54 | + - | |
| 55 | + set -e |
| 56 | + |
| 57 | + echo "Starting Automated Restore Controller (Dynamic Mode)..." |
| 58 | + |
| 59 | + # Configuration: List of Critical PVCs to Restore |
| 60 | + # Format: "Namespace/PVC-Name" |
| 61 | + # We only restore these if they don't exist yet. |
| 62 | + CRITICAL_PVCS=( |
| 63 | + "karakeep/data-pvc" |
| 64 | + "home-assistant/home-assistant-config" |
| 65 | + "immich/immich-library" |
| 66 | + "n8n/n8n-data" |
| 67 | + "paperless-ngx/paperless-data-pvc" |
| 68 | + "paperless-ngx/paperless-media-pvc" |
| 69 | + ) |
| 70 | + |
| 71 | + # Wait for Longhorn CRDs |
| 72 | + echo "Waiting for BackupVolumes CRD..." |
| 73 | + kubectl wait --for=condition=established --timeout=60s crd/backupvolumes.longhorn.io || echo "CRD not ready yet, proceeding anyway..." |
| 74 | +
|
| 75 | + # Wait for BackupVolumes to be populated from S3 |
| 76 | + echo "Waiting for BackupVolumes to sync from S3..." |
| 77 | + for i in {1..30}; do |
| 78 | + COUNT=$(kubectl get backupvolumes -n longhorn-system --no-headers 2>/dev/null | wc -l) |
| 79 | + if [ "$COUNT" -gt "0" ]; then |
| 80 | + echo "Found $COUNT BackupVolumes. Proceeding." |
| 81 | + break |
| 82 | + fi |
| 83 | + echo "No BackupVolumes found yet. Waiting... ($i/30)" |
| 84 | + sleep 5 |
| 85 | + done |
| 86 | +
|
| 87 | + # Get all BackupVolumes with their PVC info |
| 88 | + # Format: BackupVolumeName|PVCName|Namespace |
| 89 | + echo "Building Backup Index..." |
| 90 | + # Note: We use a temporary file because bash arrays and pipes can be tricky |
| 91 | + kubectl get backupvolumes -n longhorn-system -o jsonpath='{range .items[*]}{.metadata.name}|{.status.kubernetesStatus.pvcName}|{.status.kubernetesStatus.namespace}{"\n"}{end}' > /tmp/backup_index.txt |
| 92 | + |
| 93 | + for target in "${CRITICAL_PVCS[@]}"; do |
| 94 | + IFS='/' read -r target_ns target_pvc <<< "$target" |
| 95 | + |
| 96 | + echo "------------------------------------------------" |
| 97 | + echo "Processing Target: $target_ns/$target_pvc" |
| 98 | + |
| 99 | + # Check if PV already exists for this PVC (by checking our managed label) |
| 100 | + # Or just check if a PV exists that claims this PVC? |
| 101 | + # We'll stick to our naming convention pv-restore-<app> for simplicity in this script, |
| 102 | + # but ideally we should check if the PVC is already bound. |
| 103 | + |
| 104 | + PV_NAME="pv-restore-${target_pvc}" |
| 105 | + if kubectl get pv "$PV_NAME" >/dev/null 2>&1; then |
| 106 | + echo "PV $PV_NAME already exists. Skipping." |
| 107 | + continue |
| 108 | + fi |
| 109 | +
|
| 110 | + # Find matching BackupVolume |
| 111 | + # We look for a line in index that matches |$target_pvc|$target_ns |
| 112 | + MATCH=$(grep "|${target_pvc}|${target_ns}$" /tmp/backup_index.txt | head -n 1) |
| 113 | + |
| 114 | + if [ -z "$MATCH" ]; then |
| 115 | + echo "WARNING: No backup found for $target_ns/$target_pvc. Skipping." |
| 116 | + continue |
| 117 | + fi |
| 118 | + |
| 119 | + BACKUP_VOL_NAME=$(echo "$MATCH" | cut -d'|' -f1) |
| 120 | + echo "Found matching BackupVolume: $BACKUP_VOL_NAME" |
| 121 | + |
| 122 | + # Get Latest Backup Name |
| 123 | + LATEST_BACKUP=$(kubectl get backupvolume "$BACKUP_VOL_NAME" -n longhorn-system -o jsonpath='{.status.lastBackupName}') |
| 124 | + |
| 125 | + if [ -z "$LATEST_BACKUP" ]; then |
| 126 | + echo "BackupVolume found but no backups inside. Skipping." |
| 127 | + continue |
| 128 | + fi |
| 129 | + |
| 130 | + echo "Latest Backup: $LATEST_BACKUP" |
| 131 | + |
| 132 | + # Construct Backup URL |
| 133 | + BACKUP_TARGET=$(kubectl get backuptarget default -n longhorn-system -o jsonpath='{.spec.backupTargetURL}' 2>/dev/null || echo "") |
| 134 | + if [ -z "$BACKUP_TARGET" ]; then |
| 135 | + BACKUP_TARGET=$(kubectl get cm longhorn-backup-config -n longhorn-system -o jsonpath='{.data.backup-target}') |
| 136 | + fi |
| 137 | + [[ "${BACKUP_TARGET}" != */ ]] && BACKUP_TARGET="${BACKUP_TARGET}/" |
| 138 | + |
| 139 | + FULL_BACKUP_URL="${BACKUP_TARGET}?backup=${LATEST_BACKUP}&volume=${BACKUP_VOL_NAME}" |
| 140 | + |
| 141 | + # Create PV |
| 142 | + echo "Creating PV $PV_NAME..." |
| 143 | + cat <<EOF | kubectl apply -f - |
| 144 | + apiVersion: v1 |
| 145 | + kind: PersistentVolume |
| 146 | + metadata: |
| 147 | + name: $PV_NAME |
| 148 | + labels: |
| 149 | + restored-from: "$LATEST_BACKUP" |
| 150 | + managed-by: restore-controller |
| 151 | + spec: |
| 152 | + capacity: |
| 153 | + storage: 10Gi # Defaulting to 10Gi, Longhorn will resize if needed/possible |
| 154 | + accessModes: |
| 155 | + - ReadWriteOnce |
| 156 | + persistentVolumeReclaimPolicy: Retain |
| 157 | + storageClassName: longhorn |
| 158 | + csi: |
| 159 | + driver: driver.longhorn.io |
| 160 | + fsType: ext4 |
| 161 | + volumeAttributes: |
| 162 | + numberOfReplicas: "3" |
| 163 | + staleReplicaTimeout: "30" |
| 164 | + fromBackup: "$FULL_BACKUP_URL" |
| 165 | + claimRef: |
| 166 | + name: $target_pvc |
| 167 | + namespace: $target_ns |
| 168 | + EOF |
| 169 | + |
| 170 | + # Ensure Namespace exists |
| 171 | + kubectl create ns "$target_ns" --dry-run=client -o yaml | kubectl apply -f - |
| 172 | + |
| 173 | + echo "Restored $target_ns/$target_pvc from $LATEST_BACKUP" |
| 174 | + done |
| 175 | + |
| 176 | + echo "Restore process completed." |
0 commit comments