Skip to content

Commit b4964f2

Browse files
adskyiprogeradskyiproger
authored
Merge branch 'develop' into ocrvs-9792
2 parents 34b0118 + d59d835 commit b4964f2

File tree

4 files changed

+111
-92
lines changed

4 files changed

+111
-92
lines changed

‎.github/workflows/deploy-opencrvs.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,17 +100,18 @@ jobs:
100100
-f environments/${ENV}/opencrvs-services/values.yaml \
101101
--create-namespace \
102102
--atomic \
103+
--debug \
103104
--wait \
104105
--wait-for-jobs \
105106
--set image.tag="$CORE_IMAGE_TAG" \
106107
--set countryconfig.image.tag="$COUNTRYCONFIG_IMAGE_TAG" \
107108
--set countryconfig.image.name="$COUNTRYCONFIG_IMAGE_NAME" \
108-
--set hostname=${{ vars.DOMAIN }}
109+
--set hostname=${{ vars.DOMAIN }} | sed '/USER-SUPPLIED VALUES:/,$d'
109110
- name: Cleanup Helm Locks
110111
if: failure() || cancelled()
111112
run: |
112113
kubectl -n "opencrvs-${ENV}" get secrets -l owner=helm -o json | \
113-
jq -r '.items[] | select(.metadata.labels.status=="pending-install" or .metadata.labels.status=="pending-upgrade") | .metadata.name' | \
114+
jq -r '.items[] | select(.metadata.labels.status=="pending-install" or .metadata.labels.status=="pending-upgrade" or .metadata.labels.status=="pending-rollback") | .metadata.name' | \
114115
xargs -r kubectl -n "opencrvs-${ENV}" delete secret || \
115116
echo "No helm locks found, all is good"
116117
reset-data:

‎.github/workflows/k8s-reindex.yml‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,24 @@ jobs:
3838
.service.port = (.service.port | tostring)
3939
else . end
4040
' > ${namespace}.json
41-
- name: Reindex data
41+
- name: Create job elasticsearch-reindex from helm template and apply it
4242
run: |
43-
kubectl delete job -n ${namespace} elasticsearch-reindex || true
44-
helm template -f ${namespace}.json \
45-
--namespace ${namespace} \
46-
-s templates/elasticsearch-reindex.yaml \
47-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} -f -
48-
sleep 30;
49-
kubectl logs job/elasticsearch-reindex -f -n ${namespace} || true
50-
kubectl wait --for=condition=complete job/elasticsearch-reindex -n ${namespace} --timeout=600s;
43+
kubectl delete job -n ${namespace} --ignore-not-found=true elasticsearch-reindex
44+
helm template -f ${namespace}.json \
45+
--namespace ${namespace} \
46+
-s templates/elasticsearch-reindex-job.yaml \
47+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply --wait -n ${namespace} -f -
48+
- name: Checking elasticsearch-reindex job status
49+
run: |
50+
while true; do
51+
kubectl wait --for=condition=ready pod -ljob-name=elasticsearch-reindex --timeout=300s -n ${namespace} && \
52+
kubectl logs job/elasticsearch-reindex --all-containers -f -n ${namespace} && \
53+
touch /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt || break;
54+
sleep 10; done &
55+
echo "---------------------- Waiting for job completion ----------------------"
56+
kubectl wait --for=condition=complete job/elasticsearch-reindex -n ${namespace} --timeout=600s; status=$? || true
57+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
58+
[ ! -f /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt ] && kubectl logs job/elasticsearch-reindex --all-containers -n ${namespace} || \
59+
rm -vf /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt
60+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
61+
exit $status

‎.github/workflows/k8s-reset-data.yml‎

Lines changed: 64 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
11
name: Reset environment
22
run-name: "Reset ${{ inputs.environment }} environment"
3-
# FIXME:
4-
# - replace sleep 30 with kubectl wait for job completion
5-
# - add status code checks
6-
# - refactor multiple git hub steps into job list
7-
# instead of having multiple similar steps we may have a matrix job with max-parallel: 1
83
on:
94
workflow_dispatch:
105
inputs:
116
environment:
127
description: "Target environment"
138
required: true
14-
default: "dev"
159
type: choice
1610
options:
1711
- ""
@@ -20,8 +14,9 @@ on:
2014
environment:
2115
type: string
2216
jobs:
23-
reset:
24-
environment: ${{ inputs.environment }}
17+
prepare:
18+
outputs:
19+
values-file: ${{ steps.get-values.outputs.values-file }}
2520
env:
2621
namespace: opencrvs-${{ inputs.environment }}
2722
runs-on:
@@ -30,77 +25,76 @@ jobs:
3025
- ${{ inputs.environment }}
3126
steps:
3227
- name: Get helm release values and Quote specific fields that are commonly numeric
28+
id: get-values
3329
run: |
3430
helm get values opencrvs -n ${namespace} -ojson | \
3531
jq '
32+
# Quote image tags
3633
if has("image") and (.image | has("tag")) and (.image.tag | type == "number") then
3734
.image.tag = (.image.tag | tostring)
3835
else . end |
36+
37+
# Quote version numbers
3938
if has("version") and (.version | type == "number") then
4039
.version = (.version | tostring)
4140
else . end |
41+
42+
# Quote port numbers if needed (optional)
4243
if has("service") and (.service | has("port")) and (.service.port | type == "number") then
4344
.service.port = (.service.port | tostring)
4445
else . end
45-
' > ${namespace}.json
46-
- name: Cleanup environment
47-
run: |
48-
kubectl delete job -n ${namespace} --ignore-not-found=true data-cleanup
49-
helm template -f ${namespace}.json \
50-
--set data_cleanup.enabled=true \
51-
--namespace ${namespace} \
52-
-s templates/data-cleanup-job.yaml \
53-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
54-
kubectl wait --for=condition=complete job/data-cleanup -n ${namespace} --timeout=600s || true
55-
kubectl logs job/data-cleanup -f --all-containers=true -n ${namespace} || true
56-
- name: Re-run postgres on-update-core
57-
run: |
58-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-on-update-core;
59-
helm template -f ${namespace}.json \
60-
--namespace ${namespace} \
61-
-s templates/postgres-on-update-core.yaml \
62-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
63-
kubectl wait --for=condition=complete job/postgres-on-update-core -n ${namespace} --timeout=600s || true
64-
kubectl logs job/postgres-on-update-core -f --all-containers=true -n ${namespace} || true
65-
- name: Re-run postgres-data-migration
66-
run: |
67-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-data-migration;
68-
helm template -f ${namespace}.json \
69-
--namespace ${namespace} \
70-
-s templates/postgres-migration-job.yaml \
71-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
72-
kubectl wait --for=condition=complete job/postgres-data-migration -n ${namespace} --timeout=600s || true
73-
kubectl logs job/postgres-data-migration -f --all-containers=true -n ${namespace} || true
74-
- name: Re-run postgres on-update-analytics
75-
run: |
76-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-on-update-analytics;
77-
helm template -f ${namespace}.json \
78-
-s templates/postgres-on-update-analytics.yaml \
79-
--namespace ${namespace} \
80-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
81-
kubectl wait --for=condition=complete job/postgres-on-update-analytics -n ${namespace} --timeout=600s || true
82-
kubectl logs job/postgres-on-update-analytics -f --all-containers=true -n ${namespace} || true
83-
- name: Migration
84-
run: |
85-
kubectl delete job -n ${namespace} --ignore-not-found=true data-migration-on-reset
86-
helm template -f ${namespace}.json \
87-
--set data_migration.job_name=data-migration-on-reset \
88-
-s templates/data-migration-job.yaml \
89-
--namespace ${namespace} \
90-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
91-
kubectl wait --for=condition=complete job/data-migration-on-reset -n ${namespace} --timeout=600s || true
92-
kubectl logs job/data-migration-on-reset -f -n ${namespace} || true
93-
- name: Seeding data
94-
run: |
95-
kubectl delete job -n ${namespace} --ignore-not-found=true data-seed
96-
kubectl delete pod -n ${namespace} -lapp=events;
97-
kubectl wait --for=condition=ready pod -n ${namespace} -lapp=events --timeout=600s;
98-
helm template -f ${namespace}.json \
99-
--set data_seed.enabled=true \
100-
--namespace ${namespace} \
101-
-s templates/data-seed-job.yaml \
102-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait -f -
103-
sleep 10;
104-
kubectl wait --for=condition=complete job/data-seed -n ${namespace} --timeout=600s || true
105-
kubectl logs job/data-seed -f -n ${namespace} || true
106-
kubectl delete pod -n ${namespace} -lapp=events;
46+
' > /tmp/${namespace}.json
47+
echo "values-file=/tmp/${namespace}.json" >> $GITHUB_OUTPUT
48+
- name: Upload helm release values file /tmp/opencrvs-${{ inputs.environment }}.json
49+
uses: actions/upload-artifact@v4
50+
with:
51+
name: opencrvs-${{ inputs.environment }}-values-file
52+
path: /tmp/opencrvs-${{ inputs.environment }}.json
53+
retention-days: 1
54+
reset:
55+
name: ${{ matrix.job-name }}
56+
needs: prepare
57+
env:
58+
namespace: opencrvs-${{ inputs.environment }}
59+
runs-on: [self-hosted, k8s, e2e]
60+
strategy:
61+
max-parallel: 1 # Ensure jobs run one by one
62+
fail-fast: true # Stop on first failure
63+
matrix:
64+
job-name:
65+
- data-cleanup
66+
- postgres-on-update-core
67+
- postgres-data-migration
68+
- postgres-on-update-analytics
69+
- data-migration
70+
- data-seed
71+
- elasticsearch-reindex
72+
steps:
73+
- name: Download helm release values file into /tmp/opencrvs-${{ inputs.environment }}.json
74+
uses: actions/download-artifact@v4
75+
with:
76+
name: opencrvs-${{ inputs.environment }}-values-file
77+
path: /tmp
78+
- name: Create job ${{ matrix.job-name }} from helm template and apply it
79+
run: |
80+
kubectl delete job -n ${namespace} --ignore-not-found=true ${{ matrix.job-name }}
81+
helm template -f ${{ needs.prepare.outputs.values-file }} \
82+
--set data_cleanup.enabled=true \
83+
--set data_seed.enabled=true \
84+
--namespace ${namespace} \
85+
-s templates/${{ matrix.job-name }}-job.yaml \
86+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
87+
- name: Checking ${{ matrix.job-name }} job status
88+
run: |
89+
while true; do
90+
kubectl wait --for=condition=ready pod -ljob-name=${{ matrix.job-name }} --timeout=300s -n ${namespace} && \
91+
kubectl logs job/${{ matrix.job-name }} --all-containers -f -n ${namespace} && \
92+
touch /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt || break;
93+
sleep 1; done &
94+
echo "---------------------- Waiting for job completion ----------------------"
95+
kubectl wait --for=condition=complete job/${{ matrix.job-name }} -n ${namespace} --timeout=600s; status=$? || true
96+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
97+
[ ! -f /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt ] && kubectl logs job/${{ matrix.job-name }} --all-containers -n ${namespace} || \
98+
rm -vf /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt
99+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
100+
exit $status

‎.github/workflows/k8s-seed-data.yml‎

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ on:
1616
type: string
1717
jobs:
1818
seed:
19-
environment: ${{ inputs.environment }}
2019
env:
2120
namespace: opencrvs-${{ inputs.environment }}
2221
runs-on:
@@ -28,25 +27,39 @@ jobs:
2827
run: |
2928
helm get values opencrvs -n ${namespace} -ojson | \
3029
jq '
30+
# Quote image tags
3131
if has("image") and (.image | has("tag")) and (.image.tag | type == "number") then
3232
.image.tag = (.image.tag | tostring)
3333
else . end |
34+
# Quote version numbers
3435
if has("version") and (.version | type == "number") then
3536
.version = (.version | tostring)
3637
else . end |
38+
# Quote port numbers if needed (optional)
3739
if has("service") and (.service | has("port")) and (.service.port | type == "number") then
3840
.service.port = (.service.port | tostring)
3941
else . end
4042
' > ${namespace}.json
4143
- name: Seeding data
4244
run: |
43-
kubectl delete job --wait -n ${namespace} data-seed || true
44-
helm template -f ${namespace}.json \
45-
--set data_seed.enabled=true \
46-
--namespace ${namespace} \
47-
-s templates/data-seed-job.yaml \
48-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait --timeout=600s -f -
49-
kubectl get pods -n ${namespace} -lapp=data-seed
50-
kubectl wait --for=condition=Complete job/data-seed -n ${namespace} --timeout=600s; RES=$?
51-
kubectl logs job/data-seed -f -n ${namespace}
52-
exit $RES
45+
kubectl delete job -n ${namespace} --ignore-not-found=true data-seed
46+
helm template -f ${namespace}.json \
47+
--set data_seed.enabled=true \
48+
--namespace ${namespace} \
49+
-s templates/data-seed-job.yaml \
50+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply --wait -n ${namespace} -f -
51+
52+
- name: Checking data-seed job status
53+
run: |
54+
while true; do
55+
kubectl wait --for=condition=ready pod -ljob-name=data-seed --timeout=300s -n ${namespace} && \
56+
kubectl logs job/data-seed --all-containers -f -n ${namespace} && \
57+
touch /tmp/logs_stramed-${namespace}-data-seed.txt || break;
58+
sleep 10; done &
59+
echo "---------------------- Waiting for job completion ----------------------"
60+
kubectl wait --for=condition=complete job/data-seed -n ${namespace} --timeout=600s; status=$? || true
61+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
62+
[ ! -f /tmp/logs_stramed-${namespace}-data-seed.txt ] && kubectl logs job/data-seed --all-containers -n ${namespace} || \
63+
rm -vf /tmp/logs_stramed-${namespace}-data-seed.txt
64+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
65+
exit $status

0 commit comments

Comments
 (0)