Skip to content

Commit caf33bf

Browse files
adskyiprogeradskyiproger
committed
fix: Tiltfile refactoring to improve performance and reduce maintenance
1 parent d1f4a93 commit caf33bf

File tree

23 files changed

+479
-274
lines changed

23 files changed

+479
-274
lines changed

‎.github/workflows/deploy-mosip.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Deploy MOSIP
2+
run-name: "Deploy MOSIP on ${{ inputs.environment }}"
3+
on:
4+
workflow_call:
5+
inputs:
6+
environment:
7+
type: string
8+
workflow_dispatch:
9+
inputs:
10+
environment:
11+
description: "Target environment"
12+
required: true
13+
default: "dev"
14+
type: choice
15+
options:
16+
- demo
17+
- dev
18+
- dev-stg
19+
jobs:
20+
github-to-k8s-sync-env:
21+
uses: ./.github/workflows/github-to-k8s-sync-env.yml
22+
with:
23+
environment: ${{ inputs.environment }}
24+
secrets: inherit
25+
deploy:
26+
needs: github-to-k8s-sync-env
27+
environment: ${{ inputs.environment }}
28+
env:
29+
ENV: ${{ inputs.environment }}
30+
BRANCH: ${{ github.ref_name }}
31+
runs-on:
32+
- self-hosted
33+
- k8s
34+
- ${{ inputs.environment }}
35+
steps:
36+
- uses: actions/checkout@v5
37+
- name: Deploy OpenCRVS MOSIP API
38+
if: inputs.deploy_mosip
39+
run: |
40+
helm upgrade --install mosip-api oci://ghcr.io/opencrvs/opencrvs-mosip \
41+
--namespace "opencrvs-${ENV}" \
42+
-f environments/${ENV}/mosip-api/values.yaml \
43+
--set hostname=${{ vars.DOMAIN }} \
44+
--create-namespace \
45+
--atomic
46+
- name: Cleanup Helm Locks
47+
if: failure() || cancelled()
48+
run: |
49+
kubectl -n "opencrvs-${ENV}" get secrets -l owner=helm -o json | \
50+
jq -r '.items[] | select(.metadata.labels.status=="pending-install" or .metadata.labels.status=="pending-upgrade") | .metadata.name' | \
51+
xargs -r kubectl -n "opencrvs-${ENV}" delete secret || \
52+
echo "No helm locks found, all is good"

‎.github/workflows/deploy-opencrvs.yml‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ on:
1111
type: string
1212
reset:
1313
type: boolean
14-
deploy_mosip:
15-
type: boolean
1614
workflow_dispatch:
1715
inputs:
1816
core-image-tag:
@@ -37,11 +35,6 @@ on:
3735
required: false
3836
default: false
3937
type: boolean
40-
deploy_mosip:
41-
description: "Deploy MOSIP integration"
42-
required: false
43-
default: false
44-
type: boolean
4538
jobs:
4639
github-to-k8s-sync-env:
4740
uses: ./.github/workflows/github-to-k8s-sync-env.yml
@@ -86,15 +79,8 @@ jobs:
8679
EOF
8780
)
8881
echo "$SUMMARY" | sed 's/^ //' >> $GITHUB_STEP_SUMMARY
89-
- name: Deploy OpenCRVS MOSIP API
90-
if: inputs.deploy_mosip
91-
run: |
92-
helm upgrade --install mosip-api oci://ghcr.io/opencrvs/opencrvs-mosip \
93-
--namespace "opencrvs-${ENV}" \
94-
-f environments/${ENV}/mosip-api/values.yaml \
95-
--set hostname=${{ vars.DOMAIN }} \
96-
--create-namespace \
97-
--atomic
82+
- name: Create namespace
83+
run: kubectl create namespace "opencrvs-${ENV}" || true
9884
- name: Copy secrets from dependencies into application namespace
9985
# Only redis secret for now needs to be copied
10086
run: |
@@ -116,17 +102,18 @@ jobs:
116102
-f environments/${ENV}/opencrvs-services/values.yaml \
117103
--create-namespace \
118104
--atomic \
105+
--debug \
119106
--wait \
120107
--wait-for-jobs \
121108
--set image.tag="$CORE_IMAGE_TAG" \
122109
--set countryconfig.image.tag="$COUNTRYCONFIG_IMAGE_TAG" \
123110
--set countryconfig.image.name="$COUNTRYCONFIG_IMAGE_NAME" \
124-
--set hostname=${{ vars.DOMAIN }}
111+
--set hostname=${{ vars.DOMAIN }} | sed '/USER-SUPPLIED VALUES:/,$d'
125112
- name: Cleanup Helm Locks
126113
if: failure() || cancelled()
127114
run: |
128115
kubectl -n "opencrvs-${ENV}" get secrets -l owner=helm -o json | \
129-
jq -r '.items[] | select(.metadata.labels.status=="pending-install" or .metadata.labels.status=="pending-upgrade") | .metadata.name' | \
116+
jq -r '.items[] | select(.metadata.labels.status=="pending-install" or .metadata.labels.status=="pending-upgrade" or .metadata.labels.status=="pending-rollback") | .metadata.name' | \
130117
xargs -r kubectl -n "opencrvs-${ENV}" delete secret || \
131118
echo "No helm locks found, all is good"
132119
reset-data:

‎.github/workflows/k8s-reindex.yml‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,24 @@ jobs:
4040
.service.port = (.service.port | tostring)
4141
else . end
4242
' > ${namespace}.json
43-
- name: Reindex data
43+
- name: Create job elasticsearch-reindex from helm template and apply it
4444
run: |
45-
kubectl delete job -n ${namespace} elasticsearch-reindex || true
46-
helm template -f ${namespace}.json \
47-
--namespace ${namespace} \
48-
-s templates/elasticsearch-reindex.yaml \
49-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} -f -
50-
sleep 30;
51-
kubectl logs job/elasticsearch-reindex -f -n ${namespace} || true
52-
kubectl wait --for=condition=complete job/elasticsearch-reindex -n ${namespace} --timeout=600s;
45+
kubectl delete job -n ${namespace} --ignore-not-found=true elasticsearch-reindex
46+
helm template -f ${namespace}.json \
47+
--namespace ${namespace} \
48+
-s templates/elasticsearch-reindex-job.yaml \
49+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply --wait -n ${namespace} -f -
50+
- name: Checking elasticsearch-reindex job status
51+
run: |
52+
while true; do
53+
kubectl wait --for=condition=ready pod -ljob-name=elasticsearch-reindex --timeout=300s -n ${namespace} && \
54+
kubectl logs job/elasticsearch-reindex --all-containers -f -n ${namespace} && \
55+
touch /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt || break;
56+
sleep 10; done &
57+
echo "---------------------- Waiting for job completion ----------------------"
58+
kubectl wait --for=condition=complete job/elasticsearch-reindex -n ${namespace} --timeout=600s; status=$? || true
59+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
60+
[ ! -f /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt ] && kubectl logs job/elasticsearch-reindex --all-containers -n ${namespace} || \
61+
rm -vf /tmp/logs_stramed-${namespace}-elasticsearch-reindex.txt
62+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
63+
exit $status

‎.github/workflows/k8s-reset-data.yml‎

Lines changed: 64 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,11 @@
11
name: Reset environment
22
run-name: "Reset ${{ inputs.environment }} environment"
3-
# FIXME:
4-
# - replace sleep 30 with kubectl wait for job completion
5-
# - add status code checks
6-
# - refactor multiple git hub steps into job list
7-
# instead of having multiple similar steps we may have a matrix job with max-parallel: 1
83
on:
94
workflow_dispatch:
105
inputs:
116
environment:
127
description: "Target environment"
138
required: true
14-
default: "dev"
159
type: choice
1610
options:
1711
- demo
@@ -22,8 +16,9 @@ on:
2216
environment:
2317
type: string
2418
jobs:
25-
reset:
26-
environment: ${{ inputs.environment }}
19+
prepare:
20+
outputs:
21+
values-file: ${{ steps.get-values.outputs.values-file }}
2722
env:
2823
namespace: opencrvs-${{ inputs.environment }}
2924
runs-on:
@@ -32,77 +27,76 @@ jobs:
3227
- ${{ inputs.environment }}
3328
steps:
3429
- name: Get helm release values and Quote specific fields that are commonly numeric
30+
id: get-values
3531
run: |
3632
helm get values opencrvs -n ${namespace} -ojson | \
3733
jq '
34+
# Quote image tags
3835
if has("image") and (.image | has("tag")) and (.image.tag | type == "number") then
3936
.image.tag = (.image.tag | tostring)
4037
else . end |
38+
39+
# Quote version numbers
4140
if has("version") and (.version | type == "number") then
4241
.version = (.version | tostring)
4342
else . end |
43+
44+
# Quote port numbers if needed (optional)
4445
if has("service") and (.service | has("port")) and (.service.port | type == "number") then
4546
.service.port = (.service.port | tostring)
4647
else . end
47-
' > ${namespace}.json
48-
- name: Cleanup environment
49-
run: |
50-
kubectl delete job -n ${namespace} --ignore-not-found=true data-cleanup
51-
helm template -f ${namespace}.json \
52-
--set data_cleanup.enabled=true \
53-
--namespace ${namespace} \
54-
-s templates/data-cleanup-job.yaml \
55-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
56-
kubectl wait --for=condition=complete job/data-cleanup -n ${namespace} --timeout=600s || true
57-
kubectl logs job/data-cleanup -f --all-containers=true -n ${namespace} || true
58-
- name: Re-run postgres on-update-core
59-
run: |
60-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-on-update-core;
61-
helm template -f ${namespace}.json \
62-
--namespace ${namespace} \
63-
-s templates/postgres-on-update-core.yaml \
64-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
65-
kubectl wait --for=condition=complete job/postgres-on-update-core -n ${namespace} --timeout=600s || true
66-
kubectl logs job/postgres-on-update-core -f --all-containers=true -n ${namespace} || true
67-
- name: Re-run postgres-data-migration
68-
run: |
69-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-data-migration;
70-
helm template -f ${namespace}.json \
71-
--namespace ${namespace} \
72-
-s templates/postgres-migration-job.yaml \
73-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
74-
kubectl wait --for=condition=complete job/postgres-data-migration -n ${namespace} --timeout=600s || true
75-
kubectl logs job/postgres-data-migration -f --all-containers=true -n ${namespace} || true
76-
- name: Re-run postgres on-update-analytics
77-
run: |
78-
kubectl delete job -n ${namespace} --ignore-not-found=true postgres-on-update-analytics;
79-
helm template -f ${namespace}.json \
80-
-s templates/postgres-on-update-analytics.yaml \
81-
--namespace ${namespace} \
82-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -;
83-
kubectl wait --for=condition=complete job/postgres-on-update-analytics -n ${namespace} --timeout=600s || true
84-
kubectl logs job/postgres-on-update-analytics -f --all-containers=true -n ${namespace} || true
85-
- name: Migration
86-
run: |
87-
kubectl delete job -n ${namespace} --ignore-not-found=true data-migration-on-reset
88-
helm template -f ${namespace}.json \
89-
--set data_migration.job_name=data-migration-on-reset \
90-
-s templates/data-migration-job.yaml \
91-
--namespace ${namespace} \
92-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
93-
kubectl wait --for=condition=complete job/data-migration-on-reset -n ${namespace} --timeout=600s || true
94-
kubectl logs job/data-migration-on-reset -f -n ${namespace} || true
95-
- name: Seeding data
96-
run: |
97-
kubectl delete job -n ${namespace} --ignore-not-found=true data-seed
98-
kubectl delete pod -n ${namespace} -lapp=events;
99-
kubectl wait --for=condition=ready pod -n ${namespace} -lapp=events --timeout=600s;
100-
helm template -f ${namespace}.json \
101-
--set data_seed.enabled=true \
102-
--namespace ${namespace} \
103-
-s templates/data-seed-job.yaml \
104-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait -f -
105-
sleep 10;
106-
kubectl wait --for=condition=complete job/data-seed -n ${namespace} --timeout=600s || true
107-
kubectl logs job/data-seed -f -n ${namespace} || true
108-
kubectl delete pod -n ${namespace} -lapp=events;
48+
' > /tmp/${namespace}.json
49+
echo "values-file=/tmp/${namespace}.json" >> $GITHUB_OUTPUT
50+
- name: Upload helm release values file /tmp/opencrvs-${{ inputs.environment }}.json
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: opencrvs-${{ inputs.environment }}-values-file
54+
path: /tmp/opencrvs-${{ inputs.environment }}.json
55+
retention-days: 1
56+
reset:
57+
name: ${{ matrix.job-name }}
58+
needs: prepare
59+
env:
60+
namespace: opencrvs-${{ inputs.environment }}
61+
runs-on: [self-hosted, k8s, e2e]
62+
strategy:
63+
max-parallel: 1 # Ensure jobs run one by one
64+
fail-fast: true # Stop on first failure
65+
matrix:
66+
job-name:
67+
- data-cleanup
68+
- postgres-on-update-core
69+
- postgres-data-migration
70+
- postgres-on-update-analytics
71+
- data-migration
72+
- data-seed
73+
- elasticsearch-reindex
74+
steps:
75+
- name: Download helm release values file into /tmp/opencrvs-${{ inputs.environment }}.json
76+
uses: actions/download-artifact@v4
77+
with:
78+
name: opencrvs-${{ inputs.environment }}-values-file
79+
path: /tmp
80+
- name: Create job ${{ matrix.job-name }} from helm template and apply it
81+
run: |
82+
kubectl delete job -n ${namespace} --ignore-not-found=true ${{ matrix.job-name }}
83+
helm template -f ${{ needs.prepare.outputs.values-file }} \
84+
--set data_cleanup.enabled=true \
85+
--set data_seed.enabled=true \
86+
--namespace ${namespace} \
87+
-s templates/${{ matrix.job-name }}-job.yaml \
88+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n ${namespace} --wait=true -f -
89+
- name: Checking ${{ matrix.job-name }} job status
90+
run: |
91+
while true; do
92+
kubectl wait --for=condition=ready pod -ljob-name=${{ matrix.job-name }} --timeout=300s -n ${namespace} && \
93+
kubectl logs job/${{ matrix.job-name }} --all-containers -f -n ${namespace} && \
94+
touch /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt || break;
95+
sleep 1; done &
96+
echo "---------------------- Waiting for job completion ----------------------"
97+
kubectl wait --for=condition=complete job/${{ matrix.job-name }} -n ${namespace} --timeout=600s; status=$? || true
98+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
99+
[ ! -f /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt ] && kubectl logs job/${{ matrix.job-name }} --all-containers -n ${namespace} || \
100+
rm -vf /tmp/logs_stramed-${namespace}-${{ matrix.job-name }}.txt
101+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
102+
exit $status

‎.github/workflows/k8s-seed-data.yml‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ on:
1818
type: string
1919
jobs:
2020
seed:
21-
environment: ${{ inputs.environment }}
2221
env:
2322
namespace: opencrvs-${{ inputs.environment }}
2423
runs-on:
@@ -30,28 +29,39 @@ jobs:
3029
run: |
3130
helm get values opencrvs -n ${namespace} -ojson | \
3231
jq '
32+
# Quote image tags
3333
if has("image") and (.image | has("tag")) and (.image.tag | type == "number") then
3434
.image.tag = (.image.tag | tostring)
3535
else . end |
36+
# Quote version numbers
3637
if has("version") and (.version | type == "number") then
3738
.version = (.version | tostring)
3839
else . end |
40+
# Quote port numbers if needed (optional)
3941
if has("service") and (.service | has("port")) and (.service.port | type == "number") then
4042
.service.port = (.service.port | tostring)
4143
else . end
4244
' > ${namespace}.json
4345
- name: Seeding data
4446
run: |
45-
kubectl delete job -n opencrvs-${ENV} data-seed || true
46-
kubectl delete pod -n opencrvs-${ENV} -lapp=events;
47-
sleep 30;
48-
kubectl wait --for=condition=ready pod -n opencrvs-${ENV} -lapp=events;
49-
helm template -f ${namespace}.json \
50-
--set data_seed.enabled=true \
51-
--namespace opencrvs-${ENV} \
52-
-s templates/data-seed-job.yaml \
53-
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n opencrvs-${ENV} -f -
54-
sleep 30;
55-
kubectl logs job/data-seed -f -n opencrvs-${ENV} || true
56-
kubectl wait --for=condition=complete job/data-seed -n opencrvs-${ENV} --timeout=600s;
57-
kubectl delete pod -n opencrvs-${ENV} -lapp=events;
47+
kubectl delete job -n ${namespace} --ignore-not-found=true data-seed
48+
helm template -f ${namespace}.json \
49+
--set data_seed.enabled=true \
50+
--namespace ${namespace} \
51+
-s templates/data-seed-job.yaml \
52+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply --wait -n ${namespace} -f -
53+
54+
- name: Checking data-seed job status
55+
run: |
56+
while true; do
57+
kubectl wait --for=condition=ready pod -ljob-name=data-seed --timeout=300s -n ${namespace} && \
58+
kubectl logs job/data-seed --all-containers -f -n ${namespace} && \
59+
touch /tmp/logs_stramed-${namespace}-data-seed.txt || break;
60+
sleep 10; done &
61+
echo "---------------------- Waiting for job completion ----------------------"
62+
kubectl wait --for=condition=complete job/data-seed -n ${namespace} --timeout=600s; status=$? || true
63+
[ $status -ne 0 ] && kubectl get pods -n ${namespace} --show-labels && kubectl describe pod -ljob-name=${job_name} -n ${namespace};
64+
[ ! -f /tmp/logs_stramed-${namespace}-data-seed.txt ] && kubectl logs job/data-seed --all-containers -n ${namespace} || \
65+
rm -vf /tmp/logs_stramed-${namespace}-data-seed.txt
66+
kill %1 2>/dev/null && echo "Stopped log streaming" || true
67+
exit $status

0 commit comments

Comments
 (0)