Skip to content

Commit 426f428

Browse files
adskyiprogeradskyiproger
committed
fix: e2e integration with Kubernetes cluster (#81)
1 parent 24db04c commit 426f428

File tree

7 files changed

+617
-20
lines changed

7 files changed

+617
-20
lines changed

‎.github/workflows/deploy-and-e2e.yml‎

Lines changed: 123 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
name: Deploy & run E2E
2-
run-name: 'E2E by ${{ github.event.client_payload.actor || github.actor }} to ${{ github.event.client_payload.stack || github.event.inputs.stack }}. Images core: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }} and farajaland: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}'
2+
run-name: 'E2E by ${{ github.event.client_payload.actor || github.actor }} to ${{ github.event.client_payload.stack || github.event.inputs.stack }}. (core: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }}, farajaland: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }})'
33
on:
44
repository_dispatch:
55
types: [run_e2e]
66
workflow_dispatch:
77
inputs:
88
core-image-tag:
9-
description: Core DockerHub image tag
9+
description: Core image tag
1010
required: true
1111
default: 'v1.8.0'
1212
countryconfig-image-tag:
@@ -26,19 +26,71 @@ on:
2626
description: e2e branch
2727
required: false
2828
default: 'develop'
29+
runtime:
30+
required: false
31+
default: 'none'
32+
type: choice
33+
options:
34+
- none
35+
- docker
36+
- k8s
37+
reset:
38+
description: reset e2e
39+
required: false
40+
default: false
41+
type: boolean
2942
concurrency:
30-
group: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
43+
group: ${{ github.event.client_payload.stack || inputs.stack }}
3144
cancel-in-progress: true
3245

3346
jobs:
3447
debug:
3548
name: Debug output
3649
runs-on: ubuntu-22.04
50+
outputs:
51+
runtime: ${{ steps.runtime.outputs.runtime }}
52+
domain: ${{ steps.runtime.outputs.domain }}
53+
keep_e2e: ${{ steps.runtime.outputs.keep_e2e }}
3754
steps:
3855
- name: Print Entire Event Payload
3956
run: |
4057
echo "${{ toJson(github.event) }}"
41-
deploy:
58+
- name: Select docker swarm or k8s
59+
# Environment is selected based on stack hash with is always constant
60+
# Only first 8 symbols are used from hash and stack_id number is generated
61+
# - Odd numbers are deployed to k8s
62+
# - Even numbers are deployed to docker swarm
63+
# E/g ocrvs-9595 has stack id 2766917431, which means docker
64+
id: runtime
65+
env:
66+
run_id: ${{ github.run_id }}
67+
runtime: ${{ github.event.client_payload.runtime || inputs.runtime }}
68+
stack: ${{ github.event.client_payload.stack || inputs.stack }}
69+
keep_e2e: ${{ github.event.client_payload.keep-e2e || inputs.keep-e2e }}
70+
run: |
71+
stack_hash=$(echo -n "$stack" | sha256sum | head -c 8)
72+
stack_id=$((16#$stack_hash))
73+
if [[ -n "$runtime" && "$runtime" != "none" ]]; then
74+
runtime="$runtime"
75+
elif (( stack_id % 2 == 0 )); then
76+
runtime="k8s"
77+
else
78+
runtime="docker"
79+
fi
80+
if [[ "$runtime" == "docker" ]]; then
81+
domain=${{ github.event.client_payload.stack || inputs.stack }}.${{ vars.DOMAIN }}
82+
else
83+
domain=${{ github.event.client_payload.stack || inputs.stack }}.k8s-e2e.${{ vars.DOMAIN }}
84+
fi
85+
echo "domain=$domain" >> $GITHUB_OUTPUT
86+
echo "runtime=$runtime" >> $GITHUB_OUTPUT
87+
echo "keep_e2e=$keep_e2e" >> $GITHUB_OUTPUT
88+
echo "domain=$domain"
89+
echo "runtime=$runtime"
90+
echo "keep_e2e=$keep_e2e"
91+
deploy-docker:
92+
needs: debug
93+
if: needs.debug.outputs.runtime == 'docker'
4294
uses: ./.github/workflows/deploy.yml
4395
with:
4496
core-image-tag: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }}
@@ -48,6 +100,16 @@ jobs:
48100
e2e_branch: ${{ github.event.client_payload.branch || github.event.inputs.branch }}
49101
reset: 'true'
50102
secrets: inherit
103+
deploy-k8s:
104+
if: needs.debug.outputs.runtime == 'k8s'
105+
needs: debug
106+
uses: ./.github/workflows/k8s-deploy.yml
107+
with:
108+
core-image-tag: ${{ github.event.client_payload.core-image-tag || inputs.core-image-tag }}
109+
countryconfig-image-tag: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
110+
environment: ${{ github.event.client_payload.stack || inputs.stack }}
111+
reset: ${{ github.event.client_payload.reset || inputs.reset || true }}
112+
secrets: inherit
51113

52114
discover-tests:
53115
name: Discover test directories
@@ -83,7 +145,7 @@ jobs:
83145
node_modules
84146
~/.cache/yarn/v6
85147
~/.cache/ms-playwright
86-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
148+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
87149
restore-keys: |
88150
${{ runner.os }}-node-
89151
@@ -101,16 +163,22 @@ jobs:
101163
node_modules
102164
~/.cache/yarn/v6
103165
~/.cache/ms-playwright
104-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
166+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
105167
restore-keys: |
106168
${{ runner.os }}-node-
107169
test:
108-
needs: [deploy, discover-tests]
109-
runs-on: ubuntu-22.04
110-
environment: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
170+
needs: [debug, deploy-docker, deploy-k8s, discover-tests]
171+
# Continue if at least one dependency succeeded (docker OR k8s ran)
172+
if: |
173+
always() && (
174+
needs.deploy-docker.result == 'success' ||
175+
needs.deploy-k8s.result == 'success'
176+
)
177+
runs-on: ubuntu-24.04
178+
environment: ${{ github.event.client_payload.stack || inputs.stack }}
111179
strategy:
112-
fail-fast: false
113180
max-parallel: 10
181+
fail-fast: false
114182
matrix:
115183
test_file: ${{ fromJson(needs.discover-tests.outputs.test_matrix) }}
116184
name: ${{ matrix.test_file }}
@@ -122,7 +190,7 @@ jobs:
122190

123191
- name: Checkout country branch
124192
run: |
125-
git checkout ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
193+
git checkout ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
126194
- name: Set up Node.js from country .nvmrc
127195
uses: actions/setup-node@v4
128196
with:
@@ -136,7 +204,7 @@ jobs:
136204
node_modules
137205
~/.cache/yarn/v6
138206
~/.cache/ms-playwright
139-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
207+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
140208
restore-keys: |
141209
${{ runner.os }}-node-
142210
@@ -150,9 +218,16 @@ jobs:
150218
max_attempts: 3
151219
command: npx playwright install --with-deps
152220
- name: Run Playwright Tests
153-
run: npx playwright test ./e2e/testcases/${{ matrix.test_file }}
221+
run: |
222+
curl -s -o $NODE_EXTRA_CA_CERTS https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem
223+
curl -s -o playwright.config.ts https://raw.githubusercontent.com/opencrvs/e2e/refs/heads/k8s-integration/playwright.config.ts
224+
npx playwright test ./e2e/testcases/print-certificate/death/36.07-validate-certify-record-page.spec.ts
154225
env:
155-
DOMAIN: '${{ github.event.client_payload.stack || inputs.stack }}.${{ vars.DOMAIN }}'
226+
DOMAIN: '${{ needs.debug.outputs.domain }}'
227+
# Allow e2e playwright API call for Lets encrypt staging SSL Certificate
228+
NODE_EXTRA_CA_CERTS: /tmp/letsencrypt-stg-root-x1.pem
229+
# Allow e2e Browser for Lets encrypt staging SSL Certificate
230+
IGNORE_CA: '1'
156231
- id: ctrf_check
157232
if: always()
158233
run: |
@@ -171,7 +246,7 @@ jobs:
171246
- uses: actions/upload-artifact@v4
172247
if: always()
173248
with:
174-
name: playwright-report-${{ github.event.client_payload.stack || github.event.inputs.stack }}-${{ steps.artifact.outputs.artifact }}-${{ github.run_id }}-${{ github.run_attempt }}
249+
name: playwright-report-${{ github.event.client_payload.stack || inputs.stack }}-${{ steps.artifact.outputs.artifact }}-${{ github.run_id }}-${{ github.run_attempt }}
175250
path: playwright-report/
176251
retention-days: 30
177252

@@ -199,13 +274,13 @@ jobs:
199274
run: |
200275
echo "result=$PREVIOUS_CONCLUSION" >> $GITHUB_OUTPUT
201276
202-
cleanup-stack:
203-
needs: [test]
277+
cleanup-stack-docker:
278+
needs: [test, debug]
204279
runs-on: ubuntu-24.04
205-
if: github.event.client_payload.keep-e2e == 'false' || github.event.inputs.keep-e2e == 'false'
280+
if: always() && needs.debug.outputs.keep_e2e == 'false' && needs.debug.outputs.runtime == 'docker'
206281
env:
207-
stack: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
208-
keep_e2e: ${{ github.event.client_payload.keep-e2e || github.event.inputs.keep-e2e }}
282+
stack: ${{ github.event.client_payload.stack || inputs.stack }}
283+
keep_e2e: ${{ github.event.client_payload.keep-e2e || inputs.keep-e2e }}
209284
steps:
210285
- uses: actions/checkout@v4
211286
- name: Read known hosts
@@ -230,3 +305,31 @@ jobs:
230305
--ssh_host=${{ vars.SSH_HOST || secrets.SSH_HOST }} \
231306
--ssh_port=${{ vars.SSH_PORT || secrets.SSH_PORT }} \
232307
--ssh_user=${{ secrets.SSH_USER }}
308+
309+
cleanup-stack-k8s:
310+
needs: [test, debug]
311+
runs-on: [self-hosted]
312+
env:
313+
ENV: ${{ github.event.client_payload.stack || inputs.stack }}
314+
if: always() && needs.debug.outputs.keep_e2e == 'false' && needs.debug.outputs.runtime == 'k8s'
315+
steps:
316+
- name: Checkout repo
317+
uses: actions/checkout@v4
318+
- name: Update k8s-env/opencrvs/values.yaml
319+
run: |
320+
sed -i -e "s#{{STACK}}#${ENV}#g" k8s-env/opencrvs/values.yaml
321+
- name: Cleanup environment
322+
run: |
323+
kubectl delete job -n opencrvs-${ENV} --ignore-not-found=true data-cleanup
324+
helm template -f k8s-env/opencrvs/values.yaml \
325+
--set data_cleanup.enabled=true \
326+
--set image.tag="$CORE_IMAGE_TAG" \
327+
--namespace opencrvs-${ENV} \
328+
-s templates/data-cleanup-job.yaml \
329+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n opencrvs-${ENV} --wait=true -f -
330+
sleep 30;
331+
kubectl logs job/data-cleanup -f --all-containers=true -n opencrvs-${ENV} || true
332+
kubectl wait --for=condition=complete job/data-cleanup -n opencrvs-${ENV} --timeout=600s;
333+
- name: Delete helm release and namespace
334+
run: |
335+
kubectl delete namespace opencrvs-${ENV}

‎.github/workflows/k8s-deploy.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: Deploy (k8s)
2+
run-name: "Deploy OpenCRVS on k8s (core: ${{ inputs.core-image-tag }}, country: ${{ inputs.countryconfig-image-tag }})"
3+
on:
4+
workflow_call:
5+
inputs:
6+
core-image-tag:
7+
type: string
8+
countryconfig-image-tag:
9+
type: string
10+
environment:
11+
type: string
12+
reset:
13+
type: boolean
14+
workflow_dispatch:
15+
inputs:
16+
core-image-tag:
17+
description: "Tag of the core image"
18+
required: true
19+
default: "develop"
20+
countryconfig-image-tag:
21+
description: "Tag of the countryconfig image"
22+
required: true
23+
default: "develop"
24+
environment:
25+
description: "Target environment"
26+
required: true
27+
default: "demo"
28+
type: string
29+
reset:
30+
description: "Reset environment after deploy"
31+
required: false
32+
default: false
33+
type: boolean
34+
35+
jobs:
36+
deploy:
37+
env:
38+
ENV: ${{ inputs.environment }}
39+
CORE_IMAGE_TAG: ${{ inputs.core-image-tag }}
40+
COUNTRYCONFIG_IMAGE_TAG: ${{ inputs.countryconfig-image-tag }}
41+
runs-on: [self-hosted]
42+
steps:
43+
# FYI: Repository is needed only due to single file: examples/dev/opencrvs-services/values.yaml
44+
- name: Checkout repo
45+
uses: actions/checkout@v4
46+
47+
- name: Print deployment parameters
48+
run: |
49+
echo "Environment: $ENV"
50+
echo "Core Image: $CORE_IMAGE_TAG"
51+
echo "Country Config Image: $COUNTRYCONFIG_IMAGE_TAG"
52+
- name: Deploy OpenCRVS MOSIP API
53+
run: |
54+
helm upgrade --install mosip-api oci://ghcr.io/opencrvs/opencrvs-mosip \
55+
--namespace "opencrvs-${ENV}" \
56+
-f k8s-env/mosip-api/values.yaml \
57+
--set hostname=$ENV.k8s-e2e.opencrvs.dev \
58+
--create-namespace
59+
- name: Copy secrets from dependencies into application namespace
60+
run: |
61+
secrets=(
62+
"elasticsearch-admin-user"
63+
"redis-opencrvs-users"
64+
"minio-opencrvs-users"
65+
"mongodb-admin-user"
66+
"postgres-admin-user"
67+
)
68+
for secret in "${secrets[@]}"; do
69+
kubectl get secret $secret -n opencrvs-deps-e2e -o yaml \
70+
| sed "s#namespace: opencrvs-deps-e2e#namespace: opencrvs-${ENV}#" \
71+
| grep -vE 'resourceVersion|uid|creationTimestamp' \
72+
| kubectl apply -n opencrvs-${ENV} -f - \
73+
|| echo "Secret $secret doesn't exist in opencrvs-deps-e2e namespace"
74+
done
75+
- name: Update k8s-env/opencrvs/values.yaml
76+
run: |
77+
sed -i -e "s#{{STACK}}#${ENV}#g" k8s-env/opencrvs/values.yaml
78+
- name: Deploy with Helm
79+
run: |
80+
helm upgrade --install opencrvs oci://ghcr.io/opencrvs/opencrvs-services \
81+
--namespace "opencrvs-${ENV}" \
82+
-f k8s-env/opencrvs/values.yaml \
83+
--create-namespace \
84+
--set image.tag="$CORE_IMAGE_TAG" \
85+
--set countryconfig.image.tag="$COUNTRYCONFIG_IMAGE_TAG" \
86+
--set hostname=$ENV.k8s-e2e.opencrvs.dev \
87+
--set environment="$ENV"
88+
89+
reset-data:
90+
if: ${{ inputs.reset }}
91+
# name: Reset environment (k8s)
92+
# run-name: "Reset environment (core: ${{ inputs.core-image-tag }})
93+
needs: deploy
94+
uses: ./.github/workflows/k8s-reset-data.yml
95+
with:
96+
environment: ${{ inputs.environment }}
97+
core-image-tag: ${{ inputs.core-image-tag }}
98+
secrets: inherit

0 commit comments

Comments
 (0)