Skip to content

Commit 9f90a83

Browse files
adskyiprogeradskyiproger
committed
fix: e2e integration with Kubernetes cluster (#81)
1 parent 24db04c commit 9f90a83

File tree

7 files changed

+663
-51
lines changed

7 files changed

+663
-51
lines changed

‎.github/workflows/deploy-and-e2e.yml‎

Lines changed: 168 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
name: Deploy & run E2E
2-
run-name: 'E2E by ${{ github.event.client_payload.actor || github.actor }} to ${{ github.event.client_payload.stack || github.event.inputs.stack }}. Images core: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }} and farajaland: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}'
2+
run-name: 'E2E by ${{ github.event.client_payload.actor || github.actor }} to ${{ github.event.client_payload.stack || github.event.inputs.stack }}. (core: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }}, farajaland: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }})'
33
on:
44
repository_dispatch:
55
types: [run_e2e]
66
workflow_dispatch:
77
inputs:
88
core-image-tag:
9-
description: Core DockerHub image tag
9+
description: Core image tag
1010
required: true
1111
default: 'v1.8.0'
1212
countryconfig-image-tag:
@@ -26,28 +26,102 @@ on:
2626
description: e2e branch
2727
required: false
2828
default: 'develop'
29+
runtime:
30+
required: false
31+
default: 'none'
32+
type: choice
33+
options:
34+
- none
35+
- docker
36+
- k8s
37+
reset:
38+
description: reset e2e
39+
required: false
40+
default: false
41+
type: boolean
2942
concurrency:
30-
group: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
43+
group: ${{ github.event.client_payload.stack || inputs.stack }}
3144
cancel-in-progress: true
3245

3346
jobs:
3447
debug:
3548
name: Debug output
3649
runs-on: ubuntu-22.04
50+
outputs:
51+
runtime: ${{ steps.runtime.outputs.runtime }}
52+
domain: ${{ steps.runtime.outputs.domain }}
53+
keep_e2e: ${{ steps.runtime.outputs.keep_e2e }}
3754
steps:
3855
- name: Print Entire Event Payload
3956
run: |
4057
echo "${{ toJson(github.event) }}"
41-
deploy:
42-
uses: ./.github/workflows/deploy.yml
58+
- name: Select docker swarm or k8s
59+
# Environment is selected based on stack hash with is always constant
60+
# Only first 8 symbols are used from hash and stack_id number is generated
61+
# - Odd numbers are deployed to k8s
62+
# - Even numbers are deployed to docker swarm
63+
# E/g ocrvs-9595 has stack id 2766917431, which means docker
64+
id: runtime
65+
env:
66+
run_id: ${{ github.run_id }}
67+
runtime: ${{ github.event.client_payload.runtime || inputs.runtime }}
68+
stack: ${{ github.event.client_payload.stack || inputs.stack }}
69+
keep_e2e: ${{ github.event.client_payload.keep-e2e || inputs.keep-e2e }}
70+
run: |
71+
stack_hash=$(echo -n "$stack" | sha256sum | head -c 8)
72+
stack_id=$((16#$stack_hash))
73+
if [[ -n "$runtime" && "$runtime" != "none" ]]; then
74+
echo "condition 1"
75+
runtime="$runtime"
76+
elif (( stack_id % 2 == 0 )); then
77+
echo "condition 2"
78+
runtime="k8s"
79+
else
80+
echo "condition else"
81+
runtime="docker"
82+
fi
83+
if [[ "$runtime" == "docker" ]]; then
84+
domain=${{ github.event.client_payload.stack || inputs.stack }}.${{ vars.DOMAIN }}
85+
else
86+
domain=${{ github.event.client_payload.stack || inputs.stack }}.k8s-e2e.${{ vars.DOMAIN }}
87+
fi
88+
echo "domain=$domain" >> $GITHUB_OUTPUT
89+
echo "runtime=$runtime" >> $GITHUB_OUTPUT
90+
echo "keep_e2e=$keep_e2e" >> $GITHUB_OUTPUT
91+
echo "domain=$domain"
92+
echo "runtime=$runtime"
93+
echo "keep_e2e=$keep_e2e"
94+
# debug2:
95+
# needs: debug
96+
# runs-on: ubuntu-24.04
97+
# steps:
98+
# - run: |
99+
# echo domain=[${{ needs.debug.outputs.domain }}]
100+
# echo runtime=[${{ needs.debug.outputs.runtime }}]
101+
# echo keep_e2e=[${{ needs.debug.outputs.keep_e2e }}]
102+
# [ "${{ needs.debug.outputs.runtime }}" == 'k8s' ] && echo 'should work'
103+
deploy-k8s:
104+
if: always() && needs.debug.outputs.runtime == 'k8s'
105+
needs: debug
106+
uses: ./.github/workflows/k8s-deploy.yml
43107
with:
44-
core-image-tag: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }}
45-
countryconfig-image-tag: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
46-
stack: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
47-
dependencies: false
48-
e2e_branch: ${{ github.event.client_payload.branch || github.event.inputs.branch }}
49-
reset: 'true'
108+
core-image-tag: ${{ github.event.client_payload.core-image-tag || inputs.core-image-tag }}
109+
countryconfig-image-tag: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
110+
environment: ${{ github.event.client_payload.stack || inputs.stack }}
111+
reset: ${{ github.event.client_payload.reset || inputs.reset }}
50112
secrets: inherit
113+
# deploy-docker:
114+
# needs: debug
115+
# if: needs.debug.outputs.runtime == 'docker'
116+
# uses: ./.github/workflows/deploy.yml
117+
# with:
118+
# core-image-tag: ${{ github.event.client_payload.core-image-tag || github.event.inputs.core-image-tag }}
119+
# countryconfig-image-tag: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
120+
# stack: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
121+
# dependencies: false
122+
# e2e_branch: ${{ github.event.client_payload.branch || github.event.inputs.branch }}
123+
# reset: 'true'
124+
# secrets: inherit
51125

52126
discover-tests:
53127
name: Discover test directories
@@ -83,7 +157,7 @@ jobs:
83157
node_modules
84158
~/.cache/yarn/v6
85159
~/.cache/ms-playwright
86-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
160+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
87161
restore-keys: |
88162
${{ runner.os }}-node-
89163
@@ -101,16 +175,23 @@ jobs:
101175
node_modules
102176
~/.cache/yarn/v6
103177
~/.cache/ms-playwright
104-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
178+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
105179
restore-keys: |
106180
${{ runner.os }}-node-
107181
test:
108-
needs: [deploy, discover-tests]
109-
runs-on: ubuntu-22.04
110-
environment: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
182+
# needs: [debug, deploy-docker, deploy-k8s, discover-tests]
183+
needs: [debug, deploy-k8s, discover-tests]
184+
# Continue if at least one dependency succeeded (docker OR k8s ran)
185+
# if: |
186+
# always() && (
187+
# needs.deploy-docker.result == 'success' ||
188+
# needs.deploy-k8s.result == 'success'
189+
# )
190+
runs-on: ubuntu-24.04
191+
environment: ${{ github.event.client_payload.stack || inputs.stack }}
111192
strategy:
112-
fail-fast: false
113193
max-parallel: 10
194+
fail-fast: false
114195
matrix:
115196
test_file: ${{ fromJson(needs.discover-tests.outputs.test_matrix) }}
116197
name: ${{ matrix.test_file }}
@@ -122,7 +203,7 @@ jobs:
122203

123204
- name: Checkout country branch
124205
run: |
125-
git checkout ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
206+
git checkout ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
126207
- name: Set up Node.js from country .nvmrc
127208
uses: actions/setup-node@v4
128209
with:
@@ -136,7 +217,7 @@ jobs:
136217
node_modules
137218
~/.cache/yarn/v6
138219
~/.cache/ms-playwright
139-
key: ${{ github.event.client_payload.countryconfig-image-tag || github.event.inputs.countryconfig-image-tag }}
220+
key: ${{ github.event.client_payload.countryconfig-image-tag || inputs.countryconfig-image-tag }}
140221
restore-keys: |
141222
${{ runner.os }}-node-
142223
@@ -150,9 +231,16 @@ jobs:
150231
max_attempts: 3
151232
command: npx playwright install --with-deps
152233
- name: Run Playwright Tests
153-
run: npx playwright test ./e2e/testcases/${{ matrix.test_file }}
234+
run: |
235+
curl -s -o $NODE_EXTRA_CA_CERTS https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem
236+
curl -s -o playwright.config.ts https://raw.githubusercontent.com/opencrvs/e2e/refs/heads/k8s-integration/playwright.config.ts
237+
npx playwright test ./e2e/testcases/print-certificate/death/36.07-validate-certify-record-page.spec.ts
154238
env:
155-
DOMAIN: '${{ github.event.client_payload.stack || inputs.stack }}.${{ vars.DOMAIN }}'
239+
DOMAIN: '${{ needs.debug.outputs.domain }}'
240+
# Allow e2e playwright API call for Lets encrypt staging SSL Certificate
241+
NODE_EXTRA_CA_CERTS: /tmp/letsencrypt-stg-root-x1.pem
242+
# Allow e2e Browser for Lets encrypt staging SSL Certificate
243+
IGNORE_CA: '1'
156244
- id: ctrf_check
157245
if: always()
158246
run: |
@@ -171,12 +259,13 @@ jobs:
171259
- uses: actions/upload-artifact@v4
172260
if: always()
173261
with:
174-
name: playwright-report-${{ github.event.client_payload.stack || github.event.inputs.stack }}-${{ steps.artifact.outputs.artifact }}-${{ github.run_id }}-${{ github.run_attempt }}
262+
name: playwright-report-${{ github.event.client_payload.stack || inputs.stack }}-${{ steps.artifact.outputs.artifact }}-${{ github.run_id }}-${{ github.run_attempt }}
175263
path: playwright-report/
176264
retention-days: 30
177265

178266
get-previous-run:
179267
runs-on: ubuntu-latest
268+
needs: [debug]
180269
outputs:
181270
previous_run_result: ${{ steps.set-output.outputs.result }}
182271

@@ -199,34 +288,62 @@ jobs:
199288
run: |
200289
echo "result=$PREVIOUS_CONCLUSION" >> $GITHUB_OUTPUT
201290
202-
cleanup-stack:
203-
needs: [test]
204-
runs-on: ubuntu-24.04
205-
if: github.event.client_payload.keep-e2e == 'false' || github.event.inputs.keep-e2e == 'false'
291+
# cleanup-stack-docker:
292+
# needs: [test, debug]
293+
# runs-on: ubuntu-24.04
294+
# if: always() && needs.debug.outputs.keep_e2e == 'false' && needs.debug.outputs.runtime == 'docker'
295+
# env:
296+
# stack: ${{ github.event.client_payload.stack || inputs.stack }}
297+
# keep_e2e: ${{ github.event.client_payload.keep-e2e || inputs.keep-e2e }}
298+
# steps:
299+
# - uses: actions/checkout@v4
300+
# - name: Read known hosts
301+
# run: |
302+
# echo "KNOWN_HOSTS<<EOF" >> $GITHUB_ENV
303+
# sed -i -e '$a\' ./infrastructure/known-hosts
304+
# cat ./infrastructure/known-hosts >> $GITHUB_ENV
305+
# echo "EOF" >> $GITHUB_ENV
306+
# - name: Install SSH Key
307+
# uses: shimataro/ssh-key-action@v2
308+
# with:
309+
# key: ${{ secrets.SSH_KEY }}
310+
# known_hosts: ${{ env.KNOWN_HOSTS }}
311+
312+
# - name: Unset KNOWN_HOSTS variable
313+
# run: |
314+
# echo "KNOWN_HOSTS=" >> $GITHUB_ENV
315+
# - name: Cleanup e2e stack
316+
# run: |
317+
# bash infrastructure/deployment/cleanup-e2e-stack.sh \
318+
# --stack=${stack} \
319+
# --ssh_host=${{ vars.SSH_HOST || secrets.SSH_HOST }} \
320+
# --ssh_port=${{ vars.SSH_PORT || secrets.SSH_PORT }} \
321+
# --ssh_user=${{ secrets.SSH_USER }}
322+
323+
cleanup-stack-k8s:
324+
needs: [test, debug]
325+
runs-on: [self-hosted]
206326
env:
207-
stack: ${{ github.event.client_payload.stack || github.event.inputs.stack }}
208-
keep_e2e: ${{ github.event.client_payload.keep-e2e || github.event.inputs.keep-e2e }}
327+
ENV: ${{ github.event.client_payload.stack || inputs.stack }}
328+
if: always() && needs.debug.outputs.keep_e2e == 'false' && needs.debug.outputs.runtime == 'k8s'
209329
steps:
210-
- uses: actions/checkout@v4
211-
- name: Read known hosts
212-
run: |
213-
echo "KNOWN_HOSTS<<EOF" >> $GITHUB_ENV
214-
sed -i -e '$a\' ./infrastructure/known-hosts
215-
cat ./infrastructure/known-hosts >> $GITHUB_ENV
216-
echo "EOF" >> $GITHUB_ENV
217-
- name: Install SSH Key
218-
uses: shimataro/ssh-key-action@v2
219-
with:
220-
key: ${{ secrets.SSH_KEY }}
221-
known_hosts: ${{ env.KNOWN_HOSTS }}
222-
223-
- name: Unset KNOWN_HOSTS variable
224-
run: |
225-
echo "KNOWN_HOSTS=" >> $GITHUB_ENV
226-
- name: Cleanup e2e stack
227-
run: |
228-
bash infrastructure/deployment/cleanup-e2e-stack.sh \
229-
--stack=${stack} \
230-
--ssh_host=${{ vars.SSH_HOST || secrets.SSH_HOST }} \
231-
--ssh_port=${{ vars.SSH_PORT || secrets.SSH_PORT }} \
232-
--ssh_user=${{ secrets.SSH_USER }}
330+
- name: Checkout repo
331+
uses: actions/checkout@v4
332+
- name: Update k8s-env/opencrvs/values.yaml
333+
run: |
334+
sed -i -e "s#{{STACK}}#${ENV}#g" k8s-env/opencrvs/values.yaml
335+
- name: Cleanup environment
336+
run: |
337+
kubectl delete job -n opencrvs-${ENV} --ignore-not-found=true data-cleanup
338+
helm template -f k8s-env/opencrvs/values.yaml \
339+
--set data_cleanup.enabled=true \
340+
--set image.tag="$CORE_IMAGE_TAG" \
341+
--namespace opencrvs-${ENV} \
342+
-s templates/data-cleanup-job.yaml \
343+
oci://ghcr.io/opencrvs/opencrvs-services | kubectl apply -n opencrvs-${ENV} --wait=true -f -
344+
sleep 30;
345+
kubectl logs job/data-cleanup -f --all-containers=true -n opencrvs-${ENV} || true
346+
kubectl wait --for=condition=complete job/data-cleanup -n opencrvs-${ENV} --timeout=600s;
347+
- name: Delete helm release and namespace
348+
run: |
349+
kubectl delete namespace opencrvs-${ENV}

0 commit comments

Comments
 (0)