Skip to content

Commit dfde20c

Browse files
cYKatherinecYKatherine
authored
UID2-5092 AKS E2E test (#157)
* Add steps to prepare for azure metadata * Rename azure to aks * Add steps to start AKS private operator * Always reply yes with the prompt * Pass in AKS operator key * Comment out setting up AKS and stuff * Fix yq issue * Add debug message * Hardcode MANAGED_IDENTITY_ID * Use python file to add env var * Update other env names * Add commands to setup proper kube config * Replace before starting * Update ROOT path * Remove unnecessary comments * Remove aks_operator_key * Move bore urls * Wait for deployment to be ready instead of svc * Remove wait command * Update port * Add cleanup step for AKS operator * Remove unnecessary outputs * Update azure-aks path * Cleanup aks cluster only when input is aks * Make python command one line * Always cleanup private operator * Throw when error adding env * Update comments * Add meaningful comments * Remove unnecessary env variables * Replace kcc-UID2-5092-aks-e2e with v3
1 parent 4894973 commit dfde20c

File tree

9 files changed

+377
-8
lines changed

9 files changed

+377
-8
lines changed

‎.github/workflows/shared-run-e2e-tests.yaml

+40-8
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ on:
5454
default: main
5555

5656
operator_type:
57-
description: The operator type [public, gcp, azure, aws]
57+
description: The operator type [public, gcp, azure, aws, aks]
5858
type: string
5959
default: public
6060
uid2_e2e_identity_scope:
@@ -169,7 +169,7 @@ jobs:
169169
170170
- name: Bring up bore
171171
id: bore
172-
if: ${{ inputs.operator_type == 'gcp' || inputs.operator_type == 'azure' || inputs.operator_type == 'aws' || inputs.operator_type == 'eks'}}
172+
if: ${{ inputs.operator_type == 'gcp' || inputs.operator_type == 'azure' || inputs.operator_type == 'aws' || inputs.operator_type == 'eks' || inputs.operator_type == 'aks'}}
173173
env:
174174
BORE_URL: ${{ secrets.BORE_URL }}
175175
BORE_SECRET: ${{ secrets.BORE_SECRET }}
@@ -211,6 +211,16 @@ jobs:
211211
admin_root: ${{ inputs.admin_root }}
212212
eks_pcr0: ${{ inputs.eks_pcr0 }}
213213

214+
- name: Prepare AKS metadata
215+
id: prepare_aks_metadata
216+
if: ${{ inputs.operator_type == 'aks' }}
217+
uses: IABTechLab/uid2-shared-actions/actions/prepare_aks_metadata@v3
218+
with:
219+
operator_image_version: ${{ inputs.operator_image_version }}
220+
admin_root: ${{ inputs.admin_root }}
221+
bore_url_core: ${{ steps.bore.outputs.BORE_URL_CORE }}
222+
bore_url_optout: ${{ steps.bore.outputs.BORE_URL_OPTOUT }}
223+
214224
- name: Bring up Docker Compose
215225
id: docker-compose
216226
env:
@@ -281,6 +291,14 @@ jobs:
281291
eks_test_cluster: ${{ inputs.eks_test_cluster }}
282292
eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}
283293

294+
- name: Start AKS private operator
295+
id: start_aks_private_operator
296+
if: ${{ inputs.operator_type == 'aks' }}
297+
uses: IABTechLab/uid2-shared-actions/actions/start_aks_private_operator@v3
298+
with:
299+
output_template_file: ${{ steps.prepare_aks_metadata.outputs.output_template_file }}
300+
azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
301+
284302
- name: Decide E2E test environment variables
285303
id: decide_env_var
286304
shell: bash
@@ -294,6 +312,7 @@ jobs:
294312
AZURE_OPERATOR_URL: ${{ steps.start_azure_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
295313
AWS_OPERATOR_URL: ${{ steps.start_aws_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
296314
EKS_OPERATOR_URL: ${{ steps.start_EKS_operator.outputs.uid2_e2e_pipeline_operator_url }}
315+
AKS_OPERATOR_URL: ${{ steps.start_aks_private_operator.outputs.uid2_e2e_pipeline_operator_url }}
297316
run: |
298317
bash uid2-shared-actions/scripts/decide_e2e_env.sh
299318
@@ -312,28 +331,34 @@ jobs:
312331
uid2_e2e_pipeline_optout_url: ${{ steps.decide_env_var.outputs.uid2_e2e_pipeline_optout_url }}
313332

314333
- name: Stop GCP private operator
315-
if: ${{ inputs.operator_type == 'gcp' }}
334+
if: always()
316335
env:
317336
GCP_PROJECT: ${{ inputs.gcp_project }}
318337
SERVICE_ACCOUNT: ${{ inputs.gcp_service_account }}
319338
GCP_INSTANCE_NAME: ${{ steps.start_gcp_private_operator.outputs.gcp_instance_name }}
320339
run: |
321-
bash uid2-shared-actions/scripts/gcp/stop_gcp_enclave.sh
340+
if [[ ${{ inputs.operator_type }} == 'gcp' ]]; then
341+
bash uid2-shared-actions/scripts/gcp/stop_gcp_enclave.sh
342+
fi
322343
323344
- name: Stop Azure private operator
324-
if: ${{ inputs.operator_type == 'azure' }}
345+
if: always()
325346
env:
326347
AZURE_CONTAINER_GROUP_NAME: ${{ steps.start_azure_private_operator.outputs.azure_container_group_name }}
327348
run: |
328-
bash uid2-shared-actions/scripts/azure/stop_azure_enclave.sh
349+
if [[ ${{ inputs.operator_type }} == 'azure' ]]; then
350+
bash uid2-shared-actions/scripts/azure/stop_azure_enclave.sh
351+
fi
329352
330353
- name: Stop AWS private operator
331-
if: ${{ inputs.operator_type == 'aws' }}
354+
if: always()
332355
env:
333356
AWS_STACK_NAME: ${{ steps.start_aws_private_operator.outputs.aws_stack_name }}
334357
AWS_REGION: ${{ inputs.aws_region }}
335358
run: |
336-
bash uid2-shared-actions/scripts/aws/stop_aws_enclave.sh
359+
if [[ ${{ inputs.operator_type }} == 'aws' ]]; then
360+
bash uid2-shared-actions/scripts/aws/stop_aws_enclave.sh
361+
fi
337362
338363
- name: Stop EKS operator
339364
if: ${{ inputs.operator_type == 'eks' }}
@@ -342,3 +367,10 @@ jobs:
342367
eks_test_cluster: ${{ inputs.eks_test_cluster }}
343368
eks_test_cluster_region: ${{ inputs.eks_test_cluster_region }}
344369
identity_scope: ${{ inputs.uid2_e2e_identity_scope }}
370+
371+
- name: Stop AKS operator
372+
if: always()
373+
run: |
374+
if [[ ${{ inputs.operator_type }} == 'aks' ]]; then
375+
bash uid2-shared-actions/scripts/aks/stop_aks_enclave.sh
376+
fi

‎actions/prepare_aks_metadata/action.yaml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Prepare AKS Metadata
2+
description: Prepares the AKS CC artifacts and enclave metadata
3+
4+
inputs:
5+
operator_image_version:
6+
description: The uid2-operator image version
7+
default: latest
8+
admin_root:
9+
description: The root path for uid2-admin folder
10+
default: uid2-admin
11+
operator_root:
12+
description: The root path for uid2-operator folder
13+
default: uid2-operator
14+
bore_url_core:
15+
description: The bore URL for core service
16+
required: true
17+
bore_url_optout:
18+
description: The bore URL for optout service
19+
required: true
20+
outputs:
21+
output_template_file:
22+
description: The output template file
23+
value: ${{ steps.enclave_artifacts.outputs.OUTPUT_TEMPLATE_FILE }}
24+
25+
runs:
26+
using: "composite"
27+
28+
steps:
29+
- name: check azure-cli version
30+
shell: bash
31+
run: |
32+
az --version
33+
34+
- name: Generate AKS enclave deployment artifacts
35+
id: enclave_artifacts
36+
shell: bash
37+
env:
38+
IMAGE_VERSION: ${{ inputs.operator_image_version }}
39+
OPERATOR_ROOT: ${{ inputs.operator_root }}
40+
BORE_URL_CORE: ${{ inputs.bore_url_core }}
41+
BORE_URL_OPTOUT: ${{ inputs.bore_url_optout }}
42+
run: |
43+
bash uid2-shared-actions/scripts/aks/prepare_aks_artifacts.sh
44+
45+
- name: Prepare AKS enclave ID
46+
id: enclave_id
47+
shell: bash
48+
env:
49+
OUTPUT_POLICY_DIGEST_FILE: ${{ steps.enclave_artifacts.outputs.OUTPUT_POLICY_DIGEST_FILE }}
50+
run: |
51+
bash uid2-shared-actions/scripts/aks/prepare_aks_enclave_id.sh
52+
53+
- name: Prepare AKS enclave metadata
54+
id: enclave_metadata
55+
shell: bash
56+
env:
57+
ADMIN_ROOT: ${{ inputs.admin_root }}
58+
ENCLAVE_ID: ${{ steps.enclave_id.outputs.ENCLAVE_ID }}
59+
ENCLAVE_PROTOCOL: azure-cc
60+
run: |
61+
bash uid2-shared-actions/scripts/save_enclave_id_to_admin.sh

‎actions/start_aks_private_operator/action.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Start AKS Private Operator
2+
description: Spins up an AKS private operator
3+
4+
inputs:
5+
output_template_file:
6+
description: The output template file
7+
required: true
8+
azure_credentials:
9+
description: The Azure credentials
10+
required: true
11+
12+
outputs:
13+
uid2_e2e_pipeline_operator_url:
14+
description: The AKS operator URL
15+
value: ${{ steps.start_aks.outputs.uid2_e2e_pipeline_operator_url }}
16+
17+
runs:
18+
using: "composite"
19+
20+
steps:
21+
- name: Log in to Azure
22+
uses: azure/login@v2
23+
with:
24+
creds: ${{ inputs.azure_credentials }}
25+
enable-AzPSSession: true
26+
27+
- name: Start AKS private operator
28+
id: start_aks
29+
shell: bash
30+
env:
31+
OUTPUT_TEMPLATE_FILE: ${{ inputs.output_template_file }}
32+
run: |
33+
bash uid2-shared-actions/scripts/aks/start_aks_enclave.sh

‎scripts/aks/add_env.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import yaml
2+
import argparse
3+
4+
def add_env_variables(yaml_file, container_name, env_vars):
5+
"""
6+
Adds multiple environment variables to a specified container in a Kubernetes YAML file.
7+
Args:
8+
yaml_file (str): Path to the YAML file.
9+
container_name (str): Name of the container to modify.
10+
env_vars (list): List of dictionaries, each containing 'name' and 'value' of an env var.
11+
"""
12+
try:
13+
with open(yaml_file, 'r') as file:
14+
data = list(yaml.safe_load_all(file))
15+
16+
modified = False
17+
18+
for doc in data:
19+
if doc and doc.get('kind') == 'Deployment':
20+
containers = doc['spec']['template']['spec']['containers']
21+
for container in containers:
22+
if container['name'] == container_name:
23+
if 'env' not in container:
24+
container['env'] = []
25+
for env_var in env_vars:
26+
container['env'].append(env_var)
27+
modified = True
28+
break
29+
30+
if modified:
31+
with open(yaml_file, 'w') as file:
32+
yaml.dump_all(data, file, default_flow_style=False)
33+
print(f"Environment variables added to container '{container_name}' in '{yaml_file}'.")
34+
else:
35+
print(f"Container '{container_name}' not found in any Deployment document.")
36+
37+
except FileNotFoundError:
38+
raise FileNotFoundError(f"Error: File '{yaml_file}' not found.")
39+
except Exception as e:
40+
raise RuntimeError(f"An error occurred: {e}") from e #Reraise with context.
41+
42+
if __name__ == "__main__":
43+
parser = argparse.ArgumentParser(description="Add multiple environment variables to a Kubernetes YAML file.")
44+
parser.add_argument("yaml_file", help="Path to the YAML file.")
45+
parser.add_argument("container_name", help="Name of the container.")
46+
parser.add_argument("env_vars", nargs="+", help="Environment variables in 'name value' pairs.")
47+
48+
args = parser.parse_args()
49+
50+
if len(args.env_vars) % 2 != 0:
51+
print("Error: Environment variables must be provided in 'name value' pairs.")
52+
exit(1)
53+
54+
env_vars = []
55+
for i in range(0, len(args.env_vars), 2):
56+
env_vars.append({'name': args.env_vars[i], 'value': args.env_vars[i + 1]})
57+
58+
add_env_variables(args.yaml_file, args.container_name, env_vars)

‎scripts/aks/prepare_aks_artifacts.sh

+104
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
# Below resources should be prepared ahead of running the E2E test.
5+
# See https://github.com/UnifiedID2/aks-demo/tree/master/vn-aks#setup-aks--node-pool
6+
export RESOURCE_GROUP="pipeline-vn-aks"
7+
export LOCATION="eastus"
8+
export VNET_NAME="pipeline-vnet"
9+
export PUBLIC_IP_ADDRESS_NAME="pipeline-public-ip"
10+
export NAT_GATEWAY_NAME="pipeline-nat-gateway"
11+
export AKS_CLUSTER_NAME="pipelinevncluster"
12+
export KEYVAULT_NAME="pipeline-vn-aks-vault"
13+
export KEYVAULT_SECRET_NAME="pipeline-vn-aks-opr-key-name"
14+
export MANAGED_IDENTITY="pipeline-vn-aks-opr-id"
15+
export AKS_NODE_RESOURCE_GROUP="MC_${RESOURCE_GROUP}_${AKS_CLUSTER_NAME}_${LOCATION}"
16+
export SUBSCRIPTION_ID="$(az account show --query id --output tsv)"
17+
export DEPLOYMENT_ENV="integ"
18+
export MANAGED_IDENTITY_ID="/subscriptions/001a3882-eb1c-42ac-9edc-5e2872a07783/resourcegroups/pipeline-vn-aks/providers/Microsoft.ManagedIdentity/userAssignedIdentities/pipeline-vn-aks-opr-id"
19+
20+
if [ -z "${IMAGE_VERSION}" ]; then
21+
echo "IMAGE_VERSION can not be empty"
22+
exit 1
23+
fi
24+
25+
if [ -z "${OPERATOR_ROOT}" ]; then
26+
echo "OPERATOR_ROOT can not be empty"
27+
exit 1
28+
fi
29+
30+
if [ -z "${BORE_URL_CORE}" ]; then
31+
echo "BORE_URL_CORE can not be empty"
32+
exit 1
33+
fi
34+
35+
if [ -z "${BORE_URL_OPTOUT}" ]; then
36+
echo "BORE_URL_OPTOUT can not be empty"
37+
exit 1
38+
fi
39+
40+
ROOT="uid2-shared-actions/scripts/aks"
41+
OUTPUT_DIR="${ROOT}/azure-aks-artifacts"
42+
43+
IMAGE="ghcr.io/iabtechlab/uid2-operator:${IMAGE_VERSION}"
44+
45+
if [ -d "${OUTPUT_DIR}" ]; then
46+
echo "${OUTPUT_DIR} exists"
47+
fi
48+
49+
INPUT_TEMPLATE_FILE="${OPERATOR_ROOT}/scripts/azure-aks/deployment/operator.yaml"
50+
OUTPUT_TEMPLATE_FILE="${OUTPUT_DIR}/operator.yaml"
51+
OUTPUT_POLICY_DIGEST_FILE="${OUTPUT_DIR}/aks-digest.txt"
52+
53+
if [[ -d ${OUTPUT_DIR} ]]; then
54+
echo "${OUTPUT_DIR} exists, skipping - this only happens during local testing"
55+
else
56+
mkdir -p ${OUTPUT_DIR}
57+
58+
# Install confcom extension, az is originally available in GitHub workflow environment
59+
az extension add --name confcom
60+
if [[ $? -ne 0 ]]; then
61+
echo "Failed to install Azure confcom extension"
62+
exit 1
63+
fi
64+
65+
# Required by az confcom
66+
sudo usermod -aG docker ${USER}
67+
if [[ $? -ne 0 ]]; then
68+
echo "Failed to add current user to Docker group"
69+
exit 1
70+
fi
71+
72+
# Generate deployment template
73+
cp ${INPUT_TEMPLATE_FILE} ${OUTPUT_TEMPLATE_FILE}
74+
sed -i "s#IMAGE_PLACEHOLDER#${IMAGE}#g" ${OUTPUT_TEMPLATE_FILE}
75+
sed -i "s#IDENTITY_PLACEHOLDER#$MANAGED_IDENTITY_ID#g" "${OUTPUT_TEMPLATE_FILE}"
76+
sed -i "s#VAULT_NAME_PLACEHOLDER#$KEYVAULT_NAME#g" "${OUTPUT_TEMPLATE_FILE}"
77+
sed -i "s#OPERATOR_KEY_SECRET_NAME_PLACEHOLDER#$KEYVAULT_SECRET_NAME#g" "${OUTPUT_TEMPLATE_FILE}"
78+
sed -i "s#DEPLOYMENT_ENVIRONMENT_PLACEHOLDER#integ#g" "${OUTPUT_TEMPLATE_FILE}"
79+
cat ${OUTPUT_TEMPLATE_FILE}
80+
81+
python3 ${ROOT}/add_env.py ${OUTPUT_TEMPLATE_FILE} uid2-operator CORE_BASE_URL http://$BORE_URL_CORE OPTOUT_BASE_URL http://$BORE_URL_OPTOUT SKIP_VALIDATIONS true
82+
cat ${OUTPUT_TEMPLATE_FILE}
83+
# --- Finished updating yaml file with resources ---
84+
if [[ $? -ne 0 ]]; then
85+
echo "Failed to pre-process template file"
86+
exit 1
87+
fi
88+
89+
# Generate policy using debug mode as we will need to override environment variables
90+
yes | az confcom acipolicygen --virtual-node-yaml ${OUTPUT_TEMPLATE_FILE} --debug-mode > ${OUTPUT_POLICY_DIGEST_FILE}
91+
if [[ $? -ne 0 ]]; then
92+
echo "Failed to generate template file"
93+
exit 1
94+
fi
95+
# The previous pipe will be stored in ${OUTPUT_POLICY_DIGEST_FILE} as well. The below command is to remove the prompt and only extract the enclave id.
96+
sed -i 's/.*(y\/n) //g' "${OUTPUT_POLICY_DIGEST_FILE}"
97+
fi
98+
99+
if [ -z "${GITHUB_OUTPUT}" ]; then
100+
echo "Not in GitHub action"
101+
else
102+
echo "OUTPUT_TEMPLATE_FILE=${OUTPUT_TEMPLATE_FILE}" >> ${GITHUB_OUTPUT}
103+
echo "OUTPUT_POLICY_DIGEST_FILE=${OUTPUT_POLICY_DIGEST_FILE}" >> ${GITHUB_OUTPUT}
104+
fi

‎scripts/aks/prepare_aks_enclave_id.sh

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
if [[ ! -f ${OUTPUT_POLICY_DIGEST_FILE} ]]; then
5+
echo "OUTPUT_POLICY_DIGEST_FILE does not exist"
6+
exit 1
7+
fi
8+
9+
AZURE_POLICY_DIGEST="$(cat ${OUTPUT_POLICY_DIGEST_FILE})"
10+
echo "AZURE_POLICY_DIGEST=${AZURE_POLICY_DIGEST}"
11+
12+
ENCLAVE_ID=${AZURE_POLICY_DIGEST}
13+
echo "ENCLAVE_ID=${ENCLAVE_ID}"
14+
15+
# export to Github output
16+
if [ -z "${GITHUB_OUTPUT}" ]; then
17+
echo "Not in GitHub action"
18+
else
19+
echo "ENCLAVE_ID=${ENCLAVE_ID}" >> ${GITHUB_OUTPUT}
20+
fi

0 commit comments

Comments
 (0)