Skip to content

Commit 2c6a7b0

Browse files
wmuldergovwmuldergov
committed
DBC22-6678: Add backup notifications with webhook
1 parent d44e67b commit 2c6a7b0

8 files changed

Lines changed: 51 additions & 12 deletions

File tree

‎.github/workflows/crunchy-dr-failover.yaml‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,8 @@ jobs:
5656
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
5757
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
5858
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
59-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
59+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
60+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
6061
6162
- name: Wait for Gold DB to shut down
6263
run: |
@@ -99,7 +100,8 @@ jobs:
99100
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
100101
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
101102
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
102-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
103+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
104+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
103105
104106
- name: Confirm Gold DR is Primary
105107
run: |
@@ -147,7 +149,8 @@ jobs:
147149
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
148150
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
149151
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
150-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
152+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
153+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
151154
152155
- name: Confirm Gold DB deleted
153156
run: |
@@ -188,7 +191,8 @@ jobs:
188191
--set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \
189192
--set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \
190193
--set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \
191-
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}"
194+
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \
195+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
192196
193197
- name: Confirm Cluster is Standby
194198
run: |
@@ -239,7 +243,8 @@ jobs:
239243
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
240244
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
241245
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
242-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
246+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
247+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
243248
244249
- name: Wait for Gold DB DR to shut down
245250
run: |
@@ -281,7 +286,8 @@ jobs:
281286
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
282287
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
283288
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
284-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
289+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
290+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
285291
286292
- name: Confirm Gold is Primary
287293
run: |
@@ -330,7 +336,8 @@ jobs:
330336
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
331337
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
332338
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
333-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
339+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
340+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
334341
335342
- name: Verify Gold DR DB deleted
336343
run: |
@@ -372,7 +379,8 @@ jobs:
372379
--set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \
373380
--set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \
374381
--set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \
375-
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}"
382+
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \
383+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
376384
377385
- name: Confirm Gold DR is Standby
378386
run: |

‎.github/workflows/crunchy.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ jobs:
5252
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
5353
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
5454
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
55-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
55+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
56+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
5657
5758
5859
deploy-golddr:
@@ -84,4 +85,5 @@ jobs:
8485
--set backup-storage.env.S3_BUCKET.value=${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }} \
8586
--set backup-storage.env.S3_ENDPOINT.value=${{ secrets.CRUNCHY_S3_ENDPOINT }} \
8687
--set backup-storage.env.S3_USER.value=${{ secrets.CRUNCHY_S3_ACCESS_KEY }} \
87-
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }}
88+
--set backup-storage.env.S3_PASSWORD.value=${{ secrets.CRUNCHY_S3_SECRET_KEY }} \
89+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}

‎infrastructure/crunchy-postgres/README.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ Assumptions:
3434
--set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \
3535
--set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \
3636
--set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \
37-
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}"
37+
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \
38+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
39+
3840
1. Confirm everything has installed as expected
3941
1. Run these commands to set DB owner in the primary pod like this
4042
1. `oc exec -it $(oc get pods -l "postgres-operator.crunchydata.com/role=master" -o jsonpath='{.items[0].metadata.name}') -- /bin/bash`
@@ -61,7 +63,8 @@ If you are setting up Crunchy in Active-Standby configuration you will also need
6163
--set backup-storage.env.S3_BUCKET.value="${{ secrets.CRUNCHY_S3_BACKUP_STORAGE_BUCKET }}" \
6264
--set backup-storage.env.S3_ENDPOINT.value="${{ secrets.CRUNCHY_S3_ENDPOINT }}" \
6365
--set backup-storage.env.S3_USER.value="${{ secrets.CRUNCHY_S3_ACCESS_KEY }}" \
64-
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}"
66+
--set backup-storage.env.S3_PASSWORD.value="${{ secrets.CRUNCHY_S3_SECRET_KEY }}" \
67+
--set backup-storage.env.WEBHOOK_URL.value=${{ secrets.CRUNCHY_WEBHOOK_URL }}
6568

6669
Due to how the clusters are setup, you will need to update the password for `ccp_monitoring` for the Standby cluster based on this documentation: https://access.crunchydata.com/documentation/postgres-operator/latest/tutorials/backups-disaster-recovery/disaster-recovery#monitoring-a-standby-cluster
6770
Here are the steps: https://access.crunchydata.com/documentation/postgres-operator/latest/guides/exporter-configuration#setting-a-custom-ccp_monitoring-password
@@ -86,6 +89,7 @@ To be able to run that workflow, ensure you have these secrets set for each envi
8689
- CRUNCHY_S3_ACCESS_KEY
8790
- CRUNCHY_S3_SECRET_KEY
8891
- CRUNCHY_USER_PASSWORD
92+
- CRUNCHY_WEBHOOK_URL
8993
- OPENSHIFT_GOLD_TOKEN
9094
- OPENSHIFT_GOLDDR_TOKEN
9195

‎infrastructure/crunchy-postgres/values-dev.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,11 @@ backup-storage:
208208
env:
209209
ENVIRONMENT_FRIENDLY_NAME:
210210
value: "DriveBC Dev Database"
211+
ENVIRONMENT_NAME:
212+
value: "dev-drivebc-db"
213+
WEBHOOK_URL:
214+
value: "" # DO NOT UPLOAD TO GITHUB.
215+
secure: true
211216
BACKUP_STRATEGY:
212217
value: "rolling"
213218
DAILY_BACKUPS:

‎infrastructure/crunchy-postgres/values-prod.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,11 @@ backup-storage:
209209
env:
210210
ENVIRONMENT_FRIENDLY_NAME:
211211
value: "DriveBC Prod Database"
212+
ENVIRONMENT_NAME:
213+
value: "prod-drivebc-db"
214+
WEBHOOK_URL:
215+
value: "" # DO NOT UPLOAD TO GITHUB.
216+
secure: true
212217
BACKUP_STRATEGY:
213218
value: "rolling"
214219
DAILY_BACKUPS:

‎infrastructure/crunchy-postgres/values-test.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,11 @@ backup-storage:
208208
env:
209209
ENVIRONMENT_FRIENDLY_NAME:
210210
value: "DriveBC Test Database"
211+
ENVIRONMENT_NAME:
212+
value: "test-drivebc-db"
213+
WEBHOOK_URL:
214+
value: "" # DO NOT UPLOAD TO GITHUB.
215+
secure: true
211216
BACKUP_STRATEGY:
212217
value: "rolling"
213218
DAILY_BACKUPS:

‎infrastructure/crunchy-postgres/values-uat.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,11 @@ backup-storage:
208208
env:
209209
ENVIRONMENT_FRIENDLY_NAME:
210210
value: "DriveBC UAT Database"
211+
ENVIRONMENT_NAME:
212+
value: "uat-drivebc-db"
213+
WEBHOOK_URL:
214+
value: "" # DO NOT UPLOAD TO GITHUB.
215+
secure: true
211216
BACKUP_STRATEGY:
212217
value: "rolling"
213218
DAILY_BACKUPS:

‎infrastructure/crunchy-postgres/values.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,11 @@ backup-storage:
209209
env:
210210
ENVIRONMENT_FRIENDLY_NAME:
211211
value: "DriveBC Dev Database"
212+
ENVIRONMENT_NAME:
213+
value: "dev-drivebc-db"
214+
WEBHOOK_URL:
215+
value: "" # DO NOT UPLOAD TO GITHUB.
216+
secure: true
212217
BACKUP_STRATEGY:
213218
value: "rolling"
214219
DAILY_BACKUPS:

0 commit comments

Comments
 (0)