Skip to content

Commit 9b90e24

Browse files
mitchrossmitchross
committed
up
1 parent c958d77 commit 9b90e24

10 files changed

Lines changed: 608 additions & 282 deletions

File tree

‎docs/storage-architecture.md‎

Lines changed: 348 additions & 163 deletions
Large diffs are not rendered by default.

‎infrastructure/controllers/kyverno/volsync-clusterpolicy.yaml‎

Lines changed: 106 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,62 @@ metadata:
66
argocd.argoproj.io/sync-wave: "2"
77
policies.kyverno.io/title: Generate VolSync Backup Resources
88
policies.kyverno.io/description: >-
9-
Automatically generates ReplicationSource and ReplicationDestination
9+
Automatically generates ExternalSecret, ReplicationSource, and ReplicationDestination
1010
for PVCs labeled with backup=hourly or backup=daily.
11-
Requires namespace annotation: volsync.backube/privileged-movers: "true"
11+
User only needs: 1) PVC with backup label, 2) namespace annotation volsync.backube/privileged-movers: "true"
1212
spec:
1313
generateExisting: true
1414
rules:
1515
# ============================================
1616
# HOURLY BACKUP TIER (Critical Apps)
1717
# ============================================
1818

19+
# Rule 0a: Generate ExternalSecret for hourly backups (S3 credentials)
20+
- name: generate-hourly-externalsecret
21+
match:
22+
any:
23+
- resources:
24+
kinds:
25+
- PersistentVolumeClaim
26+
selector:
27+
matchLabels:
28+
backup: "hourly"
29+
generate:
30+
apiVersion: external-secrets.io/v1
31+
kind: ExternalSecret
32+
name: "{{request.object.metadata.name}}-volsync-secret"
33+
namespace: "{{request.object.metadata.namespace}}"
34+
synchronize: true
35+
data:
36+
spec:
37+
refreshInterval: "1h"
38+
secretStoreRef:
39+
kind: ClusterSecretStore
40+
name: 1password
41+
target:
42+
name: "{{request.object.metadata.name}}-volsync-secret"
43+
creationPolicy: Owner
44+
template:
45+
engineVersion: v2
46+
data:
47+
RESTIC_REPOSITORY: "s3:http://192.168.10.133:30292/volsync/{{request.object.metadata.namespace}}-{{request.object.metadata.name}}"
48+
RESTIC_PASSWORD: "{{ `{{ .restic_password }}` }}"
49+
AWS_ACCESS_KEY_ID: "{{ `{{ .access_key }}` }}"
50+
AWS_SECRET_ACCESS_KEY: "{{ `{{ .secret_key }}` }}"
51+
data:
52+
- secretKey: access_key
53+
remoteRef:
54+
key: rustfs
55+
property: access_key
56+
- secretKey: secret_key
57+
remoteRef:
58+
key: rustfs
59+
property: secret_key
60+
- secretKey: restic_password
61+
remoteRef:
62+
key: rustfs
63+
property: restic_password
64+
1965
# Rule 1: Generate ReplicationSource for hourly backups
2066
- name: generate-hourly-replicationsource
2167
match:
@@ -49,6 +95,8 @@ spec:
4995
cacheStorageClassName: longhorn
5096

5197
# Rule 2: Generate ReplicationDestination for hourly backups
98+
# NOTE: synchronize: false means RD persists when PVC is deleted (for restore)
99+
# Uses manual trigger - sync-cronjob.yaml handles coordinated syncing of ALL RDs
52100
- name: generate-hourly-replicationdestination
53101
match:
54102
any:
@@ -63,14 +111,15 @@ spec:
63111
kind: ReplicationDestination
64112
name: "{{request.object.metadata.name}}-restore"
65113
namespace: "{{request.object.metadata.namespace}}"
66-
synchronize: true
114+
synchronize: false
67115
data:
68116
spec:
69117
trigger:
70-
schedule: "30 * * * *"
118+
# Manual trigger - sync-cronjob patches this every 15 min
119+
manual: restore-once
71120
restic:
72121
repository: "{{request.object.metadata.name}}-volsync-secret"
73-
copyMethod: Snapshot
122+
copyMethod: Direct
74123
volumeSnapshotClassName: longhorn
75124
storageClassName: longhorn
76125
accessModes:
@@ -87,6 +136,52 @@ spec:
87136
# DAILY BACKUP TIER (Non-Critical Apps)
88137
# ============================================
89138

139+
# Rule 0b: Generate ExternalSecret for daily backups (S3 credentials)
140+
- name: generate-daily-externalsecret
141+
match:
142+
any:
143+
- resources:
144+
kinds:
145+
- PersistentVolumeClaim
146+
selector:
147+
matchLabels:
148+
backup: "daily"
149+
generate:
150+
apiVersion: external-secrets.io/v1
151+
kind: ExternalSecret
152+
name: "{{request.object.metadata.name}}-volsync-secret"
153+
namespace: "{{request.object.metadata.namespace}}"
154+
synchronize: true
155+
data:
156+
spec:
157+
refreshInterval: "1h"
158+
secretStoreRef:
159+
kind: ClusterSecretStore
160+
name: 1password
161+
target:
162+
name: "{{request.object.metadata.name}}-volsync-secret"
163+
creationPolicy: Owner
164+
template:
165+
engineVersion: v2
166+
data:
167+
RESTIC_REPOSITORY: "s3:http://192.168.10.133:30292/volsync/{{request.object.metadata.namespace}}-{{request.object.metadata.name}}"
168+
RESTIC_PASSWORD: "{{ `{{ .restic_password }}` }}"
169+
AWS_ACCESS_KEY_ID: "{{ `{{ .access_key }}` }}"
170+
AWS_SECRET_ACCESS_KEY: "{{ `{{ .secret_key }}` }}"
171+
data:
172+
- secretKey: access_key
173+
remoteRef:
174+
key: rustfs
175+
property: access_key
176+
- secretKey: secret_key
177+
remoteRef:
178+
key: rustfs
179+
property: secret_key
180+
- secretKey: restic_password
181+
remoteRef:
182+
key: rustfs
183+
property: restic_password
184+
90185
# Rule 3: Generate ReplicationSource for daily backups
91186
- name: generate-daily-replicationsource
92187
match:
@@ -119,6 +214,8 @@ spec:
119214
cacheStorageClassName: longhorn
120215

121216
# Rule 4: Generate ReplicationDestination for daily backups
217+
# NOTE: synchronize: false means RD persists when PVC is deleted (for restore)
218+
# Uses manual trigger - sync-cronjob.yaml handles coordinated syncing of ALL RDs
122219
- name: generate-daily-replicationdestination
123220
match:
124221
any:
@@ -133,14 +230,15 @@ spec:
133230
kind: ReplicationDestination
134231
name: "{{request.object.metadata.name}}-restore"
135232
namespace: "{{request.object.metadata.namespace}}"
136-
synchronize: true
233+
synchronize: false
137234
data:
138235
spec:
139236
trigger:
140-
schedule: "30 2 * * *"
237+
# Manual trigger - sync-cronjob patches this every 15 min
238+
manual: restore-once
141239
restic:
142240
repository: "{{request.object.metadata.name}}-volsync-secret"
143-
copyMethod: Snapshot
241+
copyMethod: Direct
144242
volumeSnapshotClassName: longhorn
145243
storageClassName: longhorn
146244
accessModes:

‎infrastructure/controllers/kyverno/volsync-restore-mutate.yaml‎

Lines changed: 72 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,112 @@
11
apiVersion: kyverno.io/v1
22
kind: ClusterPolicy
33
metadata:
4-
name: add-volsync-restore-datasource
4+
name: volsync-auto-restore
55
annotations:
6-
argocd.argoproj.io/sync-wave: "2"
7-
policies.kyverno.io/title: Add VolSync dataSourceRef for Automatic Restore
6+
# NOTE: No sync-wave needed - mutate policies are webhooks that intercept
7+
# CREATE requests, they don't run during ArgoCD sync phases
8+
policies.kyverno.io/title: Auto-Restore PVCs from VolSync Backups
9+
policies.kyverno.io/category: Backup
10+
policies.kyverno.io/severity: medium
11+
policies.kyverno.io/subject: PersistentVolumeClaim
812
policies.kyverno.io/description: >-
9-
Automatically adds dataSourceRef to PVCs with backup labels when a corresponding
10-
ReplicationDestination exists with latestImage. This enables automatic disaster
11-
recovery - if a PVC is deleted and recreated, it will restore from the latest backup.
13+
Automatically adds dataSourceRef to PVCs labeled with backup=hourly or backup=daily
14+
when a ReplicationDestination exists with a latestImage snapshot. This enables
15+
zero-touch disaster recovery - when a PVC is deleted and recreated, it automatically
16+
restores from the latest backup. For new apps (no backup exists), the PVC is created
17+
normally without dataSourceRef.
1218
spec:
19+
# This policy only applies to CREATE operations (not updates)
20+
background: false
1321
rules:
14-
- name: add-datasource-for-hourly-backups
22+
# ============================================
23+
# HOURLY BACKUP TIER - Auto Restore
24+
# ============================================
25+
- name: auto-restore-hourly-backup
1526
match:
1627
any:
1728
- resources:
1829
kinds:
1930
- PersistentVolumeClaim
31+
operations:
32+
- CREATE
2033
selector:
2134
matchLabels:
2235
backup: "hourly"
23-
# Only mutate if ReplicationDestination exists and has latestImage
24-
preconditions:
25-
all:
26-
- key: "{{request.operation}}"
27-
operator: Equals
28-
value: "CREATE"
29-
- key: "{{request.object.spec.dataSourceRef || 'null'}}"
30-
operator: Equals
31-
value: "null"
36+
# Fetch the ReplicationDestination to check for latestImage
3237
context:
33-
- name: replicationdest
38+
- name: rdName
39+
variable:
40+
value: "{{request.object.metadata.name}}-restore"
41+
- name: rdExists
42+
apiCall:
43+
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.object.metadata.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
44+
jmesPath: "metadata.name || ''"
45+
- name: latestImage
3446
apiCall:
35-
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
36-
jmesPath: "status.latestImage || 'null'"
47+
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.object.metadata.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
48+
jmesPath: "status.latestImage.name || ''"
49+
# Only mutate if:
50+
# 1. RD exists and has latestImage (backup exists to restore from)
51+
# 2. PVC doesn't already have dataSourceRef
52+
preconditions:
53+
all:
54+
- key: "{{ latestImage }}"
55+
operator: NotEquals
56+
value: ""
57+
- key: "{{ request.object.spec.dataSourceRef || '' }}"
58+
operator: AnyIn
59+
value:
60+
- ""
61+
- "null"
3762
mutate:
3863
patchStrategicMerge:
3964
spec:
4065
dataSourceRef:
4166
apiGroup: volsync.backube
4267
kind: ReplicationDestination
43-
name: "{{request.object.metadata.name}}-restore"
44-
# Only apply if latestImage exists
45-
+(dataSourceRef): "{{ replicationdest != 'null' }}"
68+
name: "{{ rdName }}"
4669

47-
- name: add-datasource-for-daily-backups
70+
# ============================================
71+
# DAILY BACKUP TIER - Auto Restore
72+
# ============================================
73+
- name: auto-restore-daily-backup
4874
match:
4975
any:
5076
- resources:
5177
kinds:
5278
- PersistentVolumeClaim
79+
operations:
80+
- CREATE
5381
selector:
5482
matchLabels:
5583
backup: "daily"
56-
# Only mutate if ReplicationDestination exists and has latestImage
57-
preconditions:
58-
all:
59-
- key: "{{request.operation}}"
60-
operator: Equals
61-
value: "CREATE"
62-
- key: "{{request.object.spec.dataSourceRef || 'null'}}"
63-
operator: Equals
64-
value: "null"
6584
context:
66-
- name: replicationdest
85+
- name: rdName
86+
variable:
87+
value: "{{request.object.metadata.name}}-restore"
88+
- name: rdExists
89+
apiCall:
90+
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.object.metadata.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
91+
jmesPath: "metadata.name || ''"
92+
- name: latestImage
6793
apiCall:
68-
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
69-
jmesPath: "status.latestImage || 'null'"
94+
urlPath: "/apis/volsync.backube/v1alpha1/namespaces/{{request.object.metadata.namespace}}/replicationdestinations/{{request.object.metadata.name}}-restore"
95+
jmesPath: "status.latestImage.name || ''"
96+
preconditions:
97+
all:
98+
- key: "{{ latestImage }}"
99+
operator: NotEquals
100+
value: ""
101+
- key: "{{ request.object.spec.dataSourceRef || '' }}"
102+
operator: AnyIn
103+
value:
104+
- ""
105+
- "null"
70106
mutate:
71107
patchStrategicMerge:
72108
spec:
73109
dataSourceRef:
74110
apiGroup: volsync.backube
75111
kind: ReplicationDestination
76-
name: "{{request.object.metadata.name}}-restore"
77-
# Only apply if latestImage exists
78-
+(dataSourceRef): "{{ replicationdest != 'null' }}"
112+
name: "{{ rdName }}"

‎infrastructure/storage/volsync/kustomization.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ namespace: volsync-system
44
resources:
55
- namespace.yaml
66
- externalsecret.yaml
7+
- sync-cronjob.yaml
78
helmCharts:
89
- name: volsync
910
repo: https://backube.github.io/helm-charts

0 commit comments

Comments
 (0)