Skip to content

Commit f23a0d8

Browse files
dbarkowskydbarkowsky
committed
potential workflow for pgbackrest restore
1 parent 6f3522d commit f23a0d8

1 file changed

Lines changed: 327 additions & 0 deletions

File tree

‎.github/workflows/pgbackrest-restore.yml‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
name: pgBackRest Database Restore
2+
3+
# Restores a PostgreSQL database from an automated pgBackRest backup.
4+
# Automates the steps in docs-md/operations/PGBACKREST_RESTORE.md.
5+
#
6+
# The annotation-based restore mechanism is used: the operator triggers
7+
# pgbackrest restore in-place without requiring a full cluster shutdown.
8+
#
9+
# Two restore options:
10+
# latest — Restore to the most recent backup point (no options needed).
11+
# backup-set — Restore from a specific named backup label. Run
12+
# `pgbackrest info --stanza=db` from the repo-host pod to list
13+
# available labels (see docs-md/operations/PGBACKREST_RESTORE.md,
14+
# Step 1).
15+
#
16+
# WARNING: This operation is destructive. All data written to the target
17+
# database after the selected backup point will be permanently lost.
18+
19+
permissions:
20+
contents: read
21+
22+
on:
23+
workflow_dispatch:
24+
inputs:
25+
environment:
26+
description: 'Target environment'
27+
required: true
28+
type: choice
29+
options:
30+
- test
31+
- prod
32+
database:
33+
description: 'Database cluster to restore'
34+
required: true
35+
type: choice
36+
options:
37+
- backend
38+
- temporal
39+
restore_type:
40+
description: 'Restore type'
41+
required: true
42+
type: choice
43+
options:
44+
- latest
45+
- backup-set
46+
backup_set:
47+
description: 'Backup set label (required when restore_type=backup-set, e.g. 20260716-020002F). Run pgbackrest info to find labels.'
48+
required: false
49+
type: string
50+
51+
jobs:
52+
restore:
53+
name: Restore ${{ inputs.database }} (${{ inputs.environment }})
54+
runs-on: ubuntu-latest
55+
permissions:
56+
contents: read
57+
environment:
58+
name: ${{ inputs.environment }}
59+
60+
steps:
61+
# -----------------------------------------------------------------------
62+
# Step 1: Validate inputs and compute resource names
63+
# -----------------------------------------------------------------------
64+
- name: Validate inputs
65+
run: |
66+
if [[ "${{ inputs.restore_type }}" == "backup-set" && -z "${{ inputs.backup_set }}" ]]; then
67+
echo "Error: backup_set is required when restore_type=backup-set"
68+
exit 1
69+
fi
70+
71+
- name: Compute resource names
72+
id: names
73+
run: |
74+
set -euo pipefail
75+
76+
ENV="${{ inputs.environment }}"
77+
DB="${{ inputs.database }}"
78+
79+
if [[ "$ENV" == "prod" ]]; then
80+
NAMESPACE="fd34fb-prod"
81+
INSTANCE_PREFIX="bcgov-di"
82+
else
83+
NAMESPACE="fd34fb-test"
84+
INSTANCE_PREFIX="bcgov-di-test"
85+
fi
86+
87+
if [[ "$DB" == "backend" ]]; then
88+
CLUSTER="${INSTANCE_PREFIX}-app-pg"
89+
else
90+
CLUSTER="${INSTANCE_PREFIX}-temporal-pg"
91+
fi
92+
93+
echo "namespace=$NAMESPACE" >> $GITHUB_OUTPUT
94+
echo "cluster=$CLUSTER" >> $GITHUB_OUTPUT
95+
echo "instance_prefix=$INSTANCE_PREFIX" >> $GITHUB_OUTPUT
96+
97+
echo "Namespace: $NAMESPACE"
98+
echo "Cluster: $CLUSTER"
99+
echo "Instance prefix: $INSTANCE_PREFIX"
100+
101+
# -----------------------------------------------------------------------
102+
# Step 2: Log in to OpenShift
103+
# -----------------------------------------------------------------------
104+
- name: Log in to OpenShift
105+
run: |
106+
oc login "${{ secrets.OPENSHIFT_SERVER }}" \
107+
--token="${{ secrets.OPENSHIFT_API_TOKEN }}" \
108+
--insecure-skip-tls-verify=true
109+
oc project "${{ steps.names.outputs.namespace }}"
110+
111+
# -----------------------------------------------------------------------
112+
# Step 3: Show available backups (informational)
113+
# -----------------------------------------------------------------------
114+
- name: Show available backups
115+
run: |
116+
NAMESPACE="${{ steps.names.outputs.namespace }}"
117+
CLUSTER="${{ steps.names.outputs.cluster }}"
118+
119+
REPO_HOST=$(oc get pods -n "$NAMESPACE" \
120+
-l "postgres-operator.crunchydata.com/cluster=${CLUSTER},postgres-operator.crunchydata.com/pgbackrest-dedicated" \
121+
-o jsonpath='{.items[0].metadata.name}')
122+
123+
echo "Repo-host pod: $REPO_HOST"
124+
echo ""
125+
echo "Available backups:"
126+
oc exec -n "$NAMESPACE" "$REPO_HOST" -c pgbackrest -- \
127+
pgbackrest info --stanza=db
128+
129+
# -----------------------------------------------------------------------
130+
# Step 4: Scale down application pods
131+
# -----------------------------------------------------------------------
132+
- name: Capture current replica counts
133+
id: replicas
134+
run: |
135+
NAMESPACE="${{ steps.names.outputs.namespace }}"
136+
PREFIX="${{ steps.names.outputs.instance_prefix }}"
137+
138+
BS=$(oc get deployment "${PREFIX}-backend-services" -n "$NAMESPACE" \
139+
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
140+
TMP=$(oc get deployment "${PREFIX}-temporal" -n "$NAMESPACE" \
141+
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
142+
TW=$(oc get deployment "${PREFIX}-temporal-worker" -n "$NAMESPACE" \
143+
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
144+
145+
echo "backend_services=$BS" >> $GITHUB_OUTPUT
146+
echo "temporal=$TMP" >> $GITHUB_OUTPUT
147+
echo "temporal_worker=$TW" >> $GITHUB_OUTPUT
148+
149+
echo "Current replicas — backend-services: $BS, temporal: $TMP, temporal-worker: $TW"
150+
151+
- name: Scale down application pods
152+
run: |
153+
NAMESPACE="${{ steps.names.outputs.namespace }}"
154+
PREFIX="${{ steps.names.outputs.instance_prefix }}"
155+
156+
# Both backend and temporal databases require temporal to be stopped
157+
# since Temporal server connects to temporal-pg.
158+
oc scale deployment "${PREFIX}-backend-services" --replicas=0 -n "$NAMESPACE" || true
159+
oc scale deployment "${PREFIX}-temporal-worker" --replicas=0 -n "$NAMESPACE" || true
160+
oc scale deployment "${PREFIX}-temporal" --replicas=0 -n "$NAMESPACE" || true
161+
162+
echo "Waiting for application pods to stop..."
163+
oc wait --for=delete pod \
164+
-l "app.kubernetes.io/instance=${PREFIX}" \
165+
-n "$NAMESPACE" \
166+
--timeout=120s 2>/dev/null || true
167+
168+
echo "Application pods stopped."
169+
170+
# -----------------------------------------------------------------------
171+
# Step 5: Apply restore spec and trigger restore
172+
# -----------------------------------------------------------------------
173+
- name: Build restore options
174+
id: opts
175+
run: |
176+
TYPE="${{ inputs.restore_type }}"
177+
SET="${{ inputs.backup_set }}"
178+
179+
if [[ "$TYPE" == "backup-set" ]]; then
180+
OPTIONS="[\"--set=${SET}\"]"
181+
else
182+
OPTIONS="[]"
183+
fi
184+
185+
echo "options=$OPTIONS" >> $GITHUB_OUTPUT
186+
echo "Restore options: $OPTIONS"
187+
188+
- name: Apply restore spec
189+
run: |
190+
NAMESPACE="${{ steps.names.outputs.namespace }}"
191+
CLUSTER="${{ steps.names.outputs.cluster }}"
192+
OPTIONS="${{ steps.opts.outputs.options }}"
193+
194+
oc patch postgrescluster "$CLUSTER" -n "$NAMESPACE" \
195+
--type=merge \
196+
-p "{\"spec\":{\"backups\":{\"pgbackrest\":{\"restore\":{\"enabled\":true,\"repoName\":\"repo1\",\"options\":${OPTIONS}}}}}}"
197+
198+
- name: Trigger restore
199+
id: trigger
200+
run: |
201+
NAMESPACE="${{ steps.names.outputs.namespace }}"
202+
CLUSTER="${{ steps.names.outputs.cluster }}"
203+
RESTORE_ID="restore-$(date '+%Y%m%d-%H%M%S')"
204+
205+
oc annotate postgrescluster "$CLUSTER" -n "$NAMESPACE" \
206+
--overwrite \
207+
"postgres-operator.crunchydata.com/pgbackrest-restore=${RESTORE_ID}"
208+
209+
echo "restore_id=$RESTORE_ID" >> $GITHUB_OUTPUT
210+
echo "Restore triggered with ID: $RESTORE_ID"
211+
212+
# -----------------------------------------------------------------------
213+
# Step 6: Wait for restore completion
214+
# -----------------------------------------------------------------------
215+
- name: Wait for restore to complete
216+
run: |
217+
NAMESPACE="${{ steps.names.outputs.namespace }}"
218+
CLUSTER="${{ steps.names.outputs.cluster }}"
219+
RESTORE_ID="${{ steps.trigger.outputs.restore_id }}"
220+
TIMEOUT=1200
221+
222+
echo "Waiting for restore ID '$RESTORE_ID' to complete (up to ${TIMEOUT}s)..."
223+
224+
while [ $TIMEOUT -gt 0 ]; do
225+
STATUS=$(oc get postgrescluster "$CLUSTER" -n "$NAMESPACE" \
226+
-o jsonpath='{.status.pgbackrest.restore}' 2>/dev/null || echo "")
227+
228+
if [ -n "$STATUS" ]; then
229+
echo "Current restore status: $STATUS"
230+
231+
CURRENT_ID=$(echo "$STATUS" | jq -r '.id // empty')
232+
FINISHED=$(echo "$STATUS" | jq -r '.finished // empty')
233+
234+
if [[ "$CURRENT_ID" == "$RESTORE_ID" ]]; then
235+
if [[ "$FINISHED" == "true" ]]; then
236+
echo "Restore completed successfully."
237+
exit 0
238+
fi
239+
fi
240+
else
241+
echo "Waiting for restore to start..."
242+
fi
243+
244+
sleep 10
245+
TIMEOUT=$((TIMEOUT - 10))
246+
done
247+
248+
echo "Error: restore timed out after 20 minutes."
249+
exit 1
250+
251+
# -----------------------------------------------------------------------
252+
# Step 7: Remove restore spec (mandatory — prevents re-restore on reconcile)
253+
# -----------------------------------------------------------------------
254+
- name: Remove restore spec
255+
if: always()
256+
run: |
257+
NAMESPACE="${{ steps.names.outputs.namespace }}"
258+
CLUSTER="${{ steps.names.outputs.cluster }}"
259+
260+
# Disable the restore spec
261+
oc patch postgrescluster "$CLUSTER" -n "$NAMESPACE" \
262+
--type=merge \
263+
-p '{"spec":{"backups":{"pgbackrest":{"restore":{"enabled":false}}}}}' \
264+
|| true
265+
266+
# Remove the trigger annotation
267+
oc annotate postgrescluster "$CLUSTER" -n "$NAMESPACE" \
268+
"postgres-operator.crunchydata.com/pgbackrest-restore-" \
269+
|| true
270+
271+
echo "Restore spec removed."
272+
273+
# -----------------------------------------------------------------------
274+
# Step 8: Verify the database is healthy
275+
# -----------------------------------------------------------------------
276+
- name: Verify database health
277+
run: |
278+
NAMESPACE="${{ steps.names.outputs.namespace }}"
279+
CLUSTER="${{ steps.names.outputs.cluster }}"
280+
281+
echo "Waiting for primary pod to become ready..."
282+
oc wait pod \
283+
-l "postgres-operator.crunchydata.com/cluster=${CLUSTER},postgres-operator.crunchydata.com/role=master" \
284+
-n "$NAMESPACE" \
285+
--for=condition=Ready \
286+
--timeout=180s
287+
288+
PG_POD=$(oc get pods -n "$NAMESPACE" \
289+
-l "postgres-operator.crunchydata.com/cluster=${CLUSTER},postgres-operator.crunchydata.com/role=master" \
290+
-o jsonpath='{.items[0].metadata.name}')
291+
292+
echo "Primary pod: $PG_POD"
293+
294+
RESULT=$(oc exec -n "$NAMESPACE" "$PG_POD" -c database -- \
295+
psql -U postgres -tAc "SELECT pg_is_in_recovery();")
296+
297+
echo "pg_is_in_recovery: $RESULT"
298+
299+
if [[ "$RESULT" == "f" ]]; then
300+
echo "Database is healthy and running as primary."
301+
else
302+
echo "Error: database is not running as primary (pg_is_in_recovery=$RESULT)."
303+
exit 1
304+
fi
305+
306+
# -----------------------------------------------------------------------
307+
# Step 9: Scale application pods back up
308+
# -----------------------------------------------------------------------
309+
- name: Scale application pods back up
310+
run: |
311+
NAMESPACE="${{ steps.names.outputs.namespace }}"
312+
PREFIX="${{ steps.names.outputs.instance_prefix }}"
313+
BS="${{ steps.replicas.outputs.backend_services }}"
314+
TMP="${{ steps.replicas.outputs.temporal }}"
315+
TW="${{ steps.replicas.outputs.temporal_worker }}"
316+
317+
# Restore to at least 1 replica even if the captured count was 0
318+
# (e.g. if pods were already down before this run).
319+
[[ "$BS" -gt 0 ]] || BS=1
320+
[[ "$TMP" -gt 0 ]] || TMP=1
321+
[[ "$TW" -gt 0 ]] || TW=1
322+
323+
oc scale deployment "${PREFIX}-backend-services" --replicas="$BS" -n "$NAMESPACE"
324+
oc scale deployment "${PREFIX}-temporal" --replicas="$TMP" -n "$NAMESPACE"
325+
oc scale deployment "${PREFIX}-temporal-worker" --replicas="$TW" -n "$NAMESPACE"
326+
327+
echo "Scaled up — backend-services: $BS, temporal: $TMP, temporal-worker: $TW"

0 commit comments

Comments
 (0)