Skip to content

Commit 448004f

Browse files
cYKatherinethomasm-ttd
cYKatherine
and
thomasm-ttd
authored
UID2-4147 Fix E2E test (#124)
* Set cluster as variable * Use branch reference * Pass cluster as inputs * Modify deployment file from uid2-operator-deployment repo * Remove uid2-operator-deployment repo as can't access private repo * Always cleanup EKS cluster * Update service and deployment name * Add `--ignore-not-found=true` * Fix test typo * Add get pods debugging message * Add compute namespace * Add delete namespace to stop eks * Fix namespace typo * Move the part of deciding env var to a separate script * Pass ini right operator url for EKS * Add http:// to localhost * Use a port that is not used * Add scripts to wait until pods are ready * Add retry count * Refactor starting EKS operator * Test with sleep infinity * Update logic * Remove sleep infinity * Put env var at the right place * Create secret in secret script * Delete github-test-secret * Add condition for uid2 and euid * Remove unnecessary cleanup * Port forward if eks cluster * Add debugging message * Update debugging message * Add logs debugging message * Lower enclave_cpu_count to 12 * Pass in eks_test_cluster_region * Pass in eks_test_cluster * source healthcheck.sh * Remove debugging message * Use bore to get eks operator url * Add http:// for eksoperator * Use network host for e2e test * Add condition for network * Add condition to e2e network * Refactor e2e_network structure * Remove unnecessary params * Test removing gh-uid2-docker * Remove github username and pat * Change logic for stopping eks operator * Remove passing in unnecessary input * Change tjm-UID2-4147-correct-eif-version-number to v3 --------- Co-authored-by: Thomas Manson <[email protected]>
1 parent b4644aa commit 448004f

File tree

10 files changed

+161
-65
lines changed

10 files changed

+161
-65
lines changed

‎.github/workflows/shared-run-e2e-tests.yaml

+25-42
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ on:
9191
eks_pcr0:
9292
description: The EKS PCR0
9393
type: string
94+
eks_test_cluster:
95+
description: The EKS Test Cluster Name
96+
type: string
97+
eks_test_cluster_region:
98+
description: The EKS Test Cluster Region
99+
type: string
100+
101+
94102

95103
env:
96104
REGISTRY: ghcr.io
@@ -217,7 +225,7 @@ jobs:
217225
BORE_URL_LOCALSTACK: ${{ steps.bore.outputs.BORE_URL_LOCALSTACK }}
218226
run: |
219227
bash uid2-shared-actions/scripts/prepare_resources_for_e2e_docker_compose.sh
220-
228+
221229
- name: Start GCP private operator
222230
id: start_gcp_private_operator
223231
if: ${{ inputs.operator_type == 'gcp' }}
@@ -266,6 +274,8 @@ jobs:
266274
operator_image_version: ${{ inputs.operator_image_version }}
267275
operator_root: ${{ inputs.operator_root }}
268276
operator_key: ${{ steps.prepare_eks_metadata.outputs.operator_key }}
277+
eks_test_cluster: ${{ inputs.eks_test_cluster }}
278+
eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}
269279

270280
- name: Decide E2E test environment variables
271281
id: decide_env_var
@@ -274,50 +284,20 @@ jobs:
274284
OPERATOR_TYPE: ${{ inputs.operator_type }}
275285
OPERATOR_URL: ${{ steps.start_gcp_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
276286
IDENTITY_SCOPE: ${{ inputs.uid2_e2e_identity_scope }}
287+
BORE_URL_CORE: ${{ steps.bore.outputs.BORE_URL_CORE }}
288+
BORE_URL_OPTOUT: ${{ steps.bore.outputs.BORE_URL_OPTOUT }}
289+
GCP_OPERATOR_URL: ${{ steps.start_gcp_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
290+
AZURE_OPERATOR_URL: ${{ steps.start_azure_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
291+
AWS_OPERATOR_URL: ${{ steps.start_aws_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
292+
EKS_OPERATOR_URL: ${{ steps.start_EKS_operator.outputs.uid2_e2e_pipeline_operator_url }}
277293
run: |
278-
if [ -z "${GITHUB_OUTPUT}" ]; then
279-
echo "Not in GitHub action"
280-
exit 1
281-
fi
282-
283-
if [ -z "${OPERATOR_TYPE}" ]; then
284-
echo "OPERATOR_TYPE not set"
285-
exit 1
286-
elif [ "${OPERATOR_TYPE}" == "public" ]; then
287-
echo "uid2_e2e_pipeline_operator_type=PUBLIC" >> ${GITHUB_OUTPUT}
288-
echo "uid2_e2e_pipeline_operator_url=http://publicoperator:8080" >> ${GITHUB_OUTPUT}
289-
echo "uid2_e2e_pipeline_operator_cloud_provider=PUBLIC" >> ${GITHUB_OUTPUT}
290-
echo "uid2_e2e_pipeline_core_url=http://core:8088" >> ${GITHUB_OUTPUT}
291-
echo "uid2_e2e_pipeline_optout_url=http://optout:8081" >> ${GITHUB_OUTPUT}
292-
else
293-
echo "uid2_e2e_pipeline_operator_type=PRIVATE" >> ${GITHUB_OUTPUT}
294-
if [ "${OPERATOR_TYPE}" == "gcp" ]; then
295-
echo "uid2_e2e_pipeline_operator_cloud_provider=GCP" >> ${GITHUB_OUTPUT}
296-
echo "uid2_e2e_pipeline_operator_url=${{ steps.start_gcp_private_operator.outputs.uid2_e2e_pipeline_operator_url }}" >> ${GITHUB_OUTPUT}
297-
elif [ "${OPERATOR_TYPE}" == "azure" ]; then
298-
echo "uid2_e2e_pipeline_operator_cloud_provider=AZURE" >> ${GITHUB_OUTPUT}
299-
echo "uid2_e2e_pipeline_operator_url=${{ steps.start_azure_private_operator.outputs.uid2_e2e_pipeline_operator_url }}" >> ${GITHUB_OUTPUT}
300-
elif [ "${OPERATOR_TYPE}" == "aws" ]; then
301-
echo "uid2_e2e_pipeline_operator_cloud_provider=AWS" >> ${GITHUB_OUTPUT}
302-
echo "uid2_e2e_pipeline_operator_url=${{ steps.start_aws_private_operator.outputs.uid2_e2e_pipeline_operator_url }}" >> ${GITHUB_OUTPUT}
303-
fi
304-
echo "uid2_e2e_pipeline_core_url=http://${{ steps.bore.outputs.BORE_URL_CORE }}" >> ${GITHUB_OUTPUT}
305-
echo "uid2_e2e_pipeline_optout_url=http://${{ steps.bore.outputs.BORE_URL_OPTOUT }}" >> ${GITHUB_OUTPUT}
306-
fi
307-
308-
if [ -z "${IDENTITY_SCOPE}" ]; then
309-
echo "IDENTITY_SCOPE not set"
310-
exit 1
311-
elif [ "${IDENTITY_SCOPE}" == "UID2" ]; then
312-
echo "uid2_e2e_phone_support=true" >> ${GITHUB_OUTPUT}
313-
elif [ "${IDENTITY_SCOPE}" == "EUID" ]; then
314-
echo "uid2_e2e_phone_support=false" >> ${GITHUB_OUTPUT}
315-
fi
294+
bash uid2-shared-actions/scripts/decide_e2e_env.sh
316295
317296
- name: Run E2E tests
318297
id: e2e
319-
uses: IABTechLab/uid2-shared-actions/actions/run_e2e_tests@v2
298+
uses: IABTechLab/uid2-shared-actions/actions/run_e2e_tests@v3
320299
with:
300+
e2e_network: ${{ steps.decide_env_var.outputs.e2e_network }}
321301
e2e_image_version: ${{ inputs.e2e_image_version }}
322302
uid2_e2e_identity_scope: ${{ inputs.uid2_e2e_identity_scope }}
323303
uid2_e2e_pipeline_operator_url: ${{ steps.decide_env_var.outputs.uid2_e2e_pipeline_operator_url }}
@@ -330,8 +310,8 @@ jobs:
330310
- name: Stop GCP private operator
331311
if: ${{ inputs.operator_type == 'gcp' }}
332312
env:
333-
GCP_PROJECT: ${{ vars.GCP_PROJECT }}
334-
SERVICE_ACCOUNT: ${{ vars.GCP_SERVICE_ACCOUNT }}
313+
GCP_PROJECT: ${{ inputs.gcp_project }}
314+
SERVICE_ACCOUNT: ${{ inputs.gcp_service_account }}
335315
GCP_INSTANCE_NAME: ${{ steps.start_gcp_private_operator.outputs.gcp_instance_name }}
336316
run: |
337317
bash uid2-shared-actions/scripts/gcp/stop_gcp_enclave.sh
@@ -354,3 +334,6 @@ jobs:
354334
- name: Stop EKS operator
355335
if: ${{ inputs.operator_type == 'eks' }}
356336
uses: IABTechLab/uid2-shared-actions/actions/stop_eks_operator@v3
337+
with:
338+
eks_test_cluster: ${{ inputs.eks_test_cluster }}
339+
eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}

‎actions/run_e2e_tests/action.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ name: Shared E2E Test
22
description: Pull uid2-e2e Docker image and run E2E test suite
33

44
inputs:
5+
e2e_network:
6+
description: The network to use for the E2E test
7+
type: string
8+
default: e2e_default
59
e2e_image_version:
610
description: The version of E2E image
711
required: false
@@ -69,5 +73,5 @@ runs:
6973
--env UID2_E2E_OPTOUT_TO_CALL_CORE_API_TOKEN='UID2-O-L-127-pDqphU.6FuXzThQMY8YEsCA8crqvAlzyGrjcF8P6XO84=' \
7074
--env UID2_E2E_CORE_URL='${{ inputs.uid2_e2e_pipeline_core_url }}' \
7175
--env UID2_E2E_OPTOUT_URL='${{ inputs.uid2_e2e_pipeline_optout_url }}' \
72-
--network e2e_default \
76+
--network '${{ inputs.e2e_network }}' \
7377
ghcr.io/iabtechlab/uid2-e2e:${{ inputs.e2e_image_version }}

‎actions/start_eks_operator/action.yaml

+23-9
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,17 @@ inputs:
2323
operator_key:
2424
description: The operator key
2525
required: true
26-
27-
# outputs:
28-
# uid2_e2e_pipeline_operator_url:
29-
# description: The EKS operator URL
30-
# value: ${{ steps.start_aws.outputs.uid2_e2e_pipeline_operator_url }}
26+
eks_test_cluster:
27+
description: The EKS Test Cluster Name
28+
required: true
29+
eks_test_cluster_region:
30+
description: The EKS Test Cluster Region
31+
required: true
32+
33+
outputs:
34+
uid2_e2e_pipeline_operator_url:
35+
description: The EKS operator URL
36+
value: ${{ steps.start_eks.outputs.uid2_e2e_pipeline_operator_url }}
3137

3238
runs:
3339
using: "composite"
@@ -36,7 +42,7 @@ runs:
3642
- name: Configure AWS credentials
3743
uses: aws-actions/configure-aws-credentials@v4
3844
with:
39-
aws-region: us-east-2
45+
aws-region: ${{ inputs.eks_test_cluster_region }}
4046
role-to-assume: arn:aws:iam::072245134533:role/github-runner-for-uid2-operator
4147

4248
- name: Check AWS caller identity
@@ -46,8 +52,13 @@ runs:
4652
- name: Set up kubernetes context
4753
shell: bash
4854
run: |
49-
aws eks --region us-east-2 update-kubeconfig --name uid-test-git-us-east-2
55+
aws eks --region ${{ inputs.eks_test_cluster_region }} update-kubeconfig --name ${{ inputs.eks_test_cluster }}
5056
57+
- name: Clean up previous run
58+
shell: bash
59+
run: |
60+
bash uid2-shared-actions/scripts/eks/stop_eks_enclave.sh
61+
5162
- name: Create secrets in the kubernetes cluster
5263
id: create_secrets
5364
shell: bash
@@ -64,12 +75,15 @@ runs:
6475
env:
6576
IMAGE_VERSION: ${{ inputs.operator_image_version }}
6677
OPERATOR_ROOT: ${{ inputs.operator_root }}
78+
IDENTITY_SCOPE: ${{ inputs.identity_scope }}
6779
run: |
6880
bash uid2-shared-actions/scripts/eks/prepare_eks_deployment_files.sh
6981
7082
- name: Start EKS operator
7183
id: start_eks
7284
shell: bash
85+
env:
86+
OPERATOR_ROOT: ${{ inputs.operator_root }}
7387
run: |
74-
cat ${{ inputs.operator_root }}/scripts/aws/eks/deployment_files/deployment.yaml
75-
kubectl apply -f ${{ inputs.operator_root }}/scripts/aws/eks/deployment_files/deployment.yaml
88+
bash uid2-shared-actions/scripts/eks/start_eks_enclave.sh
89+

‎actions/stop_eks_operator/action.yaml

+11-4
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
name: Stop EKS Operator
22

3+
inputs:
4+
eks_test_cluster:
5+
description: The EKS Test Cluster Name
6+
required: true
7+
eks_test_cluster_region:
8+
description: The EKS Test Cluster Region
9+
required: true
10+
311
runs:
412
using: "composite"
513

614
steps:
715
- name: Configure AWS credentials
816
uses: aws-actions/configure-aws-credentials@v4
917
with:
10-
aws-region: us-east-2
18+
aws-region: ${{ inputs.eks_test_cluster_region }}
1119
role-to-assume: arn:aws:iam::072245134533:role/github-runner-for-uid2-operator
1220

1321
- name: Check AWS caller identity
@@ -17,11 +25,10 @@ runs:
1725
- name: Set up kubernetes context
1826
shell: bash
1927
run: |
20-
aws eks --region us-east-2 update-kubeconfig --name uid-test-git-us-east-2
28+
aws eks --region ${{ inputs.eks_test_cluster_region }} update-kubeconfig --name ${{ inputs.eks_test_cluster }}
2129
2230
- name: Stop EKS operator
2331
id: stop_eks
2432
shell: bash
2533
run: |
26-
kubectl delete deployment uid2-operator
27-
kubectl delete secret github-test-secret
34+
bash uid2-shared-actions/scripts/eks/stop_eks_enclave.sh

‎scripts/decide_e2e_env.sh

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
if [ -z "${GITHUB_OUTPUT}" ]; then
2+
echo "Not in GitHub action"
3+
exit 1
4+
fi
5+
6+
if [ -z "${OPERATOR_TYPE}" ]; then
7+
echo "OPERATOR_TYPE not set"
8+
exit 1
9+
fi
10+
11+
if [ -z "${IDENTITY_SCOPE}" ]; then
12+
echo "IDENTITY_SCOPE not set"
13+
exit 1
14+
fi
15+
16+
if [ "${OPERATOR_TYPE}" == "eks" ]; then
17+
echo "e2e_network=host" >> ${GITHUB_OUTPUT}
18+
else
19+
echo "e2e_network=e2e_default" >> ${GITHUB_OUTPUT}
20+
fi
21+
22+
if [ "${OPERATOR_TYPE}" == "public" ]; then
23+
echo "uid2_e2e_pipeline_operator_type=PUBLIC" >> ${GITHUB_OUTPUT}
24+
echo "uid2_e2e_pipeline_operator_url=http://publicoperator:8080" >> ${GITHUB_OUTPUT}
25+
echo "uid2_e2e_pipeline_operator_cloud_provider=PUBLIC" >> ${GITHUB_OUTPUT}
26+
echo "uid2_e2e_pipeline_core_url=http://core:8088" >> ${GITHUB_OUTPUT}
27+
echo "uid2_e2e_pipeline_optout_url=http://optout:8081" >> ${GITHUB_OUTPUT}
28+
else
29+
echo "uid2_e2e_pipeline_operator_type=PRIVATE" >> ${GITHUB_OUTPUT}
30+
if [ "${OPERATOR_TYPE}" == "gcp" ]; then
31+
echo "uid2_e2e_pipeline_operator_cloud_provider=GCP" >> ${GITHUB_OUTPUT}
32+
echo "uid2_e2e_pipeline_operator_url=${GCP_OPERATOR_URL}" >> ${GITHUB_OUTPUT}
33+
elif [ "${OPERATOR_TYPE}" == "azure" ]; then
34+
echo "uid2_e2e_pipeline_operator_cloud_provider=AZURE" >> ${GITHUB_OUTPUT}
35+
echo "uid2_e2e_pipeline_operator_url=${AZURE_OPERATOR_URL}" >> ${GITHUB_OUTPUT}
36+
elif [ "${OPERATOR_TYPE}" == "aws" ]; then
37+
echo "uid2_e2e_pipeline_operator_cloud_provider=AWS" >> ${GITHUB_OUTPUT}
38+
echo "uid2_e2e_pipeline_operator_url=${AWS_OPERATOR_URL}" >> ${GITHUB_OUTPUT}
39+
elif [ "${OPERATOR_TYPE}" == "eks" ]; then
40+
echo "uid2_e2e_pipeline_operator_cloud_provider=AWS" >> ${GITHUB_OUTPUT}
41+
echo "uid2_e2e_pipeline_operator_url=${EKS_OPERATOR_URL}" >> ${GITHUB_OUTPUT}
42+
fi
43+
echo "uid2_e2e_pipeline_core_url=http://${BORE_URL_CORE}" >> ${GITHUB_OUTPUT}
44+
echo "uid2_e2e_pipeline_optout_url=http://${BORE_URL_OPTOUT}" >> ${GITHUB_OUTPUT}
45+
fi
46+
47+
if [ "${IDENTITY_SCOPE}" == "UID2" ]; then
48+
echo "uid2_e2e_phone_support=true" >> ${GITHUB_OUTPUT}
49+
elif [ "${IDENTITY_SCOPE}" == "EUID" ]; then
50+
echo "uid2_e2e_phone_support=false" >> ${GITHUB_OUTPUT}
51+
fi

‎scripts/eks/create_secret_in_k8.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ jq_string_update ${SECRET_JSON_FILE} api_token "${OPERATOR_KEY}"
2626

2727
cat ${SECRET_JSON_FILE}
2828

29-
kubectl create secret generic github-test-secret --from-file=config=uid2-shared-actions/scripts/eks/secret.json
29+
kubectl create namespace compute
30+
kubectl create secret generic github-test-secret --from-file=config=uid2-shared-actions/scripts/eks/secret.json -n compute

‎scripts/eks/prepare_eks_deployment_files.sh

+9-2
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,22 @@ if [ -z "${IMAGE_VERSION}" ]; then
1111
exit 1
1212
fi
1313

14+
if [ -z "${IDENTITY_SCOPE}" ]; then
15+
echo "IDENTITY_SCOPE can not be empty"
16+
exit 1
17+
fi
18+
1419
ROOT="."
1520
DEPLOYMENT_FILES_ROOT="${OPERATOR_ROOT}/scripts/aws/eks/deployment_files"
21+
DEPLOYMENT_FILE="${DEPLOYMENT_FILES_ROOT}/test-deployment.yaml"
1622

1723
ls -al
1824

19-
IMAGE="ghcr.io/iabtechlab/uid2-operator-eks-uid2:${IMAGE_VERSION}"
25+
IMAGE="ghcr.io/iabtechlab/uid2-operator-eks-${IDENTITY_SCOPE,,}:${IMAGE_VERSION}"
2026

21-
sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" "${DEPLOYMENT_FILES_ROOT}/deployment.yaml"
27+
sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" "${DEPLOYMENT_FILE}"
2228
if [[ $? -ne 0 ]]; then
2329
echo "Failed to pre-process deployment file"
2430
exit 1
2531
fi
32+
cat ${DEPLOYMENT_FILE}

‎scripts/eks/secret.json

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"api_token": "fake-operator-key",
3-
"service_instances": 6,
4-
"enclave_cpu_count": 6,
5-
"enclave_memory_mb": 24576,
3+
"enclave_cpu_count": "12",
4+
"enclave_memory_mb": "40152",
65
"environment": "integ",
7-
"customize_enclave": "false",
6+
"customize_enclave": "true",
87
"core_base_url": "https://core-integ.uidapi.com",
9-
"optout_base_url": "https://optout-integ.uidapi.com"
10-
}
8+
"optout_base_url": "https://optout-integ.uidapi.com",
9+
"operator_type": "private"
10+
}

‎scripts/eks/start_eks_enclave.sh

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
ROOT="uid2-shared-actions/scripts"
5+
source "${ROOT}/healthcheck.sh"
6+
7+
if [ -z "${OPERATOR_ROOT}" ]; then
8+
echo "OPERATOR_ROOT can not be empty"
9+
exit 1
10+
fi
11+
12+
cat "${OPERATOR_ROOT}/scripts/aws/eks/deployment_files/test-deployment.yaml"
13+
14+
kubectl apply -f "${OPERATOR_ROOT}/scripts/aws/eks/deployment_files/test-deployment.yaml"
15+
kubectl get pods --all-namespaces
16+
17+
kubectl get services -n compute
18+
kubectl port-forward svc/operator-service -n compute 27015:80 > /dev/null 2>&1 &
19+
EKS_OPERATOR_URL="http://localhost:27015"
20+
21+
kubectl get pods --all-namespaces
22+
HEALTHCHECK_URL="${EKS_OPERATOR_URL}/ops/healthcheck"
23+
24+
# Health check - for 5 mins
25+
healthcheck "${HEALTHCHECK_URL}" 60
26+
27+
echo "uid2_e2e_pipeline_operator_url=${EKS_OPERATOR_URL}" >> ${GITHUB_OUTPUT}

‎scripts/eks/stop_eks_enclave.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
kubectl delete namespace compute --ignore-not-found=true
2+
kubectl delete namespace nitro-enclaves --ignore-not-found=true

0 commit comments

Comments
 (0)