Skip to content

Commit 0aa18f9

Browse files
SylvainSenechalSylvainSenechal
committed
run e2e out of cluster
1 parent d3c8a15 commit 0aa18f9

File tree

9 files changed

+373
-382
lines changed

9 files changed

+373
-382
lines changed

‎.github/actions/deploy/action.yaml‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,5 @@ runs:
8787
if: ${{ inputs.deploy_metadata == 'true' }}
8888
- name: End-to-end configuration
8989
shell: bash
90-
run: bash configure-e2e.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "default"
91-
working-directory: ./.github/scripts/end2end
92-
- name: Linting
93-
shell: bash
94-
run: bash run-e2e-test.sh "end2end" ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} "lint" "default"
90+
run: bash configure-e2e.sh "end2end" "default"
9591
working-directory: ./.github/scripts/end2end

‎.github/scripts/end2end/configure-e2e-ctst.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ set -exu
33

44
DIR=$(dirname "$0")
55

6+
# Set up ingress endpoints and /etc/hosts for out-of-cluster access
7+
source "$DIR/configure-e2e-endpoints.sh"
8+
69
# Get kafka image name and tag
710
kafka_image() {
811
source <( "$DIR"/../../../solution/kafka_build_vars.sh )
@@ -76,8 +79,10 @@ UUID=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernet
7679
UUID=${UUID%.*}
7780
UUID=${UUID:1}
7881

79-
echo "127.0.0.1 iam.zenko.local s3-local-file.zenko.local keycloak.zenko.local \
80-
sts.zenko.local management.zenko.local s3.zenko.local website.mywebsite.com utilization.zenko.local" | sudo tee -a /etc/hosts
82+
# Ensure additional hostnames are in /etc/hosts
83+
if ! grep -q "s3-local-file.zenko.local" /etc/hosts 2>/dev/null; then
84+
echo "127.0.0.1 s3-local-file.zenko.local website.mywebsite.com" | sudo tee -a /etc/hosts
85+
fi
8186

8287
# Add bucket notification target
8388
envsubst < ./configs/notification_destinations.yaml | kubectl apply -f -

‎.github/scripts/end2end/configure-e2e-endpoints.sh‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
#!/bin/bash
2+
# configure-e2e-endpoints.sh
3+
#
4+
# Creates ingress resources for internal services that don't have one,
5+
# and adds all *.zenko.local hostnames to /etc/hosts.
6+
#
7+
# This allows tests to run outside the cluster (directly on the CI runner)
8+
# using the same ingress-based endpoints for all Zenko services.
9+
#
10+
# Idempotent: safe to source multiple times.
11+
#
12+
# Usage:
13+
# source configure-e2e-endpoints.sh # sets ZENKO_* endpoint vars
14+
# bash configure-e2e-endpoints.sh # just creates ingresses + /etc/hosts
15+
16+
set -eu
17+
18+
ZENKO_NAME="${ZENKO_NAME:-end2end}"
19+
NAMESPACE="${NAMESPACE:-default}"
20+
21+
# --- Create missing Ingress resources ---
22+
23+
apply_ingress() {
24+
local name="$1"
25+
local host="$2"
26+
local service="$3"
27+
28+
# Skip if an ingress already serves this host (e.g., from a prior Zenko instance in PRA)
29+
if kubectl get ingress -A -o jsonpath='{.items[*].spec.rules[*].host}' | grep -qw "${host}"; then
30+
echo "Ingress for ${host} already exists, skipping"
31+
return
32+
fi
33+
34+
kubectl apply -f - <<EOF
35+
apiVersion: networking.k8s.io/v1
36+
kind: Ingress
37+
metadata:
38+
name: ${name}
39+
namespace: ${NAMESPACE}
40+
annotations:
41+
nginx.ingress.kubernetes.io/proxy-body-size: "0"
42+
labels:
43+
app.kubernetes.io/instance: ${ZENKO_NAME}
44+
app.kubernetes.io/name: ${name##*-}
45+
spec:
46+
ingressClassName: nginx
47+
rules:
48+
- host: ${host}
49+
http:
50+
paths:
51+
- backend:
52+
service:
53+
name: ${service}
54+
port:
55+
name: http
56+
path: /
57+
pathType: Prefix
58+
EOF
59+
}
60+
61+
# Backbeat API — used by node tests (CRR) and CTST
62+
apply_ingress \
63+
"${ZENKO_NAME}-backbeat-api-ingress" \
64+
"backbeat-api.zenko.local" \
65+
"${ZENKO_NAME}-management-backbeat-api"
66+
67+
# Vault auth API — used by CTST
68+
apply_ingress \
69+
"${ZENKO_NAME}-vault-auth-api-ingress" \
70+
"vault-auth.zenko.local" \
71+
"${ZENKO_NAME}-connector-vault-auth-api"
72+
73+
# S3C (Ring) — only when metadata namespace exists (ENABLE_RING_TESTS=true)
74+
if kubectl get namespace metadata &>/dev/null; then
75+
kubectl apply -f - <<EOF
76+
apiVersion: networking.k8s.io/v1
77+
kind: Ingress
78+
metadata:
79+
name: s3c-ingress
80+
namespace: metadata
81+
annotations:
82+
nginx.ingress.kubernetes.io/proxy-body-size: "0"
83+
spec:
84+
ingressClassName: nginx
85+
rules:
86+
- host: s3c.local
87+
http:
88+
paths:
89+
- backend:
90+
service:
91+
name: s3c-cloudserver
92+
port:
93+
number: 8000
94+
path: /
95+
pathType: Prefix
96+
EOF
97+
fi
98+
99+
# --- Wait for ingress controller to pick them up ---
100+
101+
if kubectl get ingress "${ZENKO_NAME}-backbeat-api-ingress" &>/dev/null; then
102+
kubectl wait --for=jsonpath='{.status.loadBalancer.ingress}' \
103+
ingress/${ZENKO_NAME}-backbeat-api-ingress \
104+
ingress/${ZENKO_NAME}-vault-auth-api-ingress \
105+
--timeout=60s 2>/dev/null || true
106+
fi
107+
108+
if kubectl get ingress s3c-ingress -n metadata &>/dev/null; then
109+
kubectl wait --for=jsonpath='{.status.loadBalancer.ingress}' \
110+
ingress/s3c-ingress -n metadata \
111+
--timeout=60s 2>/dev/null || true
112+
fi
113+
114+
# --- /etc/hosts setup ---
115+
116+
ZENKO_HOSTS="s3.zenko.local iam.zenko.local sts.zenko.local management.zenko.local keycloak.zenko.local utilization.zenko.local backbeat-api.zenko.local vault-auth.zenko.local aws-mock.zenko.local azure-mock.zenko.local devstoreaccount1.blob.azure-mock.zenko.local devstoreaccount1.queue.azure-mock.zenko.local s3c.local"
117+
118+
if ! grep -q "backbeat-api.zenko.local" /etc/hosts 2>/dev/null; then
119+
echo "127.0.0.1 ${ZENKO_HOSTS}" | sudo tee -a /etc/hosts
120+
fi
121+
122+
# --- Export endpoint variables ---
123+
# These use the ingress hostnames, reachable from outside the cluster.
124+
125+
export CLOUDSERVER_HOST="s3.zenko.local"
126+
export CLOUDSERVER_ENDPOINT="http://s3.zenko.local"
127+
export BACKBEAT_API_ENDPOINT="http://backbeat-api.zenko.local"
128+
export VAULT_ENDPOINT="http://iam.zenko.local"
129+
export VAULT_STS_ENDPOINT="http://sts.zenko.local"
130+
export VAULT_AUTH_HOST="vault-auth.zenko.local"
131+
132+
echo "=== Endpoints configured for out-of-cluster access ==="
133+
echo " S3: ${CLOUDSERVER_ENDPOINT}"
134+
echo " Backbeat API: ${BACKBEAT_API_ENDPOINT}"
135+
echo " Vault IAM: ${VAULT_ENDPOINT}"
136+
echo " Vault STS: ${VAULT_STS_ENDPOINT}"
137+
echo " Vault Auth: http://${VAULT_AUTH_HOST}"

‎.github/scripts/end2end/configure-e2e.sh‎

Lines changed: 82 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -6,49 +6,40 @@ DIR=$(dirname "${0}")
66
. "$DIR"/common.sh
77

88
ZENKO_NAME=${1:-end2end}
9-
E2E_IMAGE=${2:-ghcr.io/scality/zenko/zenko-e2e:latest}
10-
NAMESPACE=${3:-default}
11-
12-
SERVICE_ACCOUNT="${ZENKO_NAME}-config"
13-
POD_NAME="${ZENKO_NAME}-config"
14-
MANAGEMENT_ENDPOINT="http://${ZENKO_NAME}-management-orbit-api:5001"
15-
IAM_ENDPOINT="http://${ZENKO_NAME}-management-vault-iam-admin-api"
16-
STS_ENDPOINT="http://${ZENKO_NAME}-connector-vault-sts-api"
9+
NAMESPACE=${2:-default}
10+
11+
# Set up ingress endpoints and /etc/hosts for out-of-cluster access
12+
. "$DIR/configure-e2e-endpoints.sh"
13+
14+
# For the primary Zenko instance, use the default ingress hostnames.
15+
# For secondary instances (e.g., PRA), prepare-pra.sh sets ZENKO_*_INGRESS
16+
# env vars via GITHUB_ENV (e.g., management.dr.zenko.local). The operator
17+
# creates ingresses with those hostnames. We discover them from the ingress
18+
# resources so configure-e2e.sh stays generic.
19+
if [ "${ZENKO_NAME}" = "end2end" ]; then
20+
MANAGEMENT_ENDPOINT="http://management.zenko.local"
21+
IAM_ENDPOINT="http://iam.zenko.local"
22+
STS_ENDPOINT="http://sts.zenko.local"
23+
else
24+
# Discover management/IAM/STS hostnames from the Zenko instance's ingresses
25+
MGMT_HOST=$(kubectl get ingress -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" \
26+
-o jsonpath='{.items[*].spec.rules[*].host}' | tr ' ' '\n' | grep '^management')
27+
IAM_HOST=$(kubectl get ingress -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" \
28+
-o jsonpath='{.items[*].spec.rules[*].host}' | tr ' ' '\n' | grep '^iam')
29+
STS_HOST=$(kubectl get ingress -n "${NAMESPACE}" -l "app.kubernetes.io/instance=${ZENKO_NAME}" \
30+
-o jsonpath='{.items[*].spec.rules[*].host}' | tr ' ' '\n' | grep '^sts')
31+
MANAGEMENT_ENDPOINT="http://${MGMT_HOST}"
32+
IAM_ENDPOINT="http://${IAM_HOST}"
33+
STS_ENDPOINT="http://${STS_HOST}"
34+
# Add PRA-specific hostnames to /etc/hosts (for runner access)
35+
PRA_HOSTS="${MGMT_HOST} ${IAM_HOST} ${STS_HOST}"
36+
if ! grep -q "${MGMT_HOST}" /etc/hosts 2>/dev/null; then
37+
echo "127.0.0.1 ${PRA_HOSTS}" | sudo tee -a /etc/hosts
38+
fi
39+
fi
1740
UUID=$(kubectl get zenko ${ZENKO_NAME} --namespace ${NAMESPACE} -o jsonpath='{.status.instanceID}')
1841
TOKEN=$(get_token)
1942

20-
cat <<EOF | kubectl apply -f -
21-
---
22-
apiVersion: v1
23-
kind: ServiceAccount
24-
metadata:
25-
name: ${SERVICE_ACCOUNT}
26-
---
27-
apiVersion: rbac.authorization.k8s.io/v1
28-
kind: Role
29-
metadata:
30-
name: ${SERVICE_ACCOUNT}
31-
rules:
32-
- apiGroups:
33-
- ""
34-
resources:
35-
- secrets
36-
verbs:
37-
- '*'
38-
---
39-
kind: RoleBinding
40-
apiVersion: rbac.authorization.k8s.io/v1
41-
metadata:
42-
name: ${SERVICE_ACCOUNT}
43-
subjects:
44-
- kind: ServiceAccount
45-
name: ${SERVICE_ACCOUNT}
46-
roleRef:
47-
kind: Role
48-
name: ${SERVICE_ACCOUNT}
49-
apiGroup: rbac.authorization.k8s.io
50-
EOF
51-
5243
kafka_image() {
5344
source <( "$DIR"/../../../solution/kafka_build_vars.sh )
5445
echo "$KAFKA_IMAGE:$KAFKA_TAG-$BUILD_TREE_HASH"
@@ -72,52 +63,57 @@ kubectl run kafka-topics \
7263
kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
7364
kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists"
7465

75-
kubectl run ${POD_NAME} \
76-
--image ${E2E_IMAGE} \
77-
--rm \
78-
--attach=True \
79-
--restart=Never \
80-
--pod-running-timeout=5m \
81-
--namespace=${NAMESPACE} \
82-
--image-pull-policy=IfNotPresent \
83-
--overrides="{ \"spec\": { \"serviceAccount\": \"${SERVICE_ACCOUNT}\" } }" \
84-
--env="TOKEN=${TOKEN}" \
85-
--env="UUID=${UUID}" \
86-
--env="MANAGEMENT_ENDPOINT=${MANAGEMENT_ENDPOINT}" \
87-
--env="IAM_ENDPOINT=${IAM_ENDPOINT}" \
88-
--env="STS_ENDPOINT=${STS_ENDPOINT}" \
89-
--env="NAMESPACE=${NAMESPACE}" \
90-
--env=VERIFY_CERTIFICATES=false \
91-
--env=ENABLE_RING_TESTS=${ENABLE_RING_TESTS} \
92-
--env=RING_S3C_ACCESS_KEY=${RING_S3C_ACCESS_KEY} \
93-
--env=RING_S3C_SECRET_KEY=${RING_S3C_SECRET_KEY} \
94-
--env=RING_S3C_ENDPOINT=${RING_S3C_ENDPOINT} \
95-
--env=RING_S3C_BACKEND_SOURCE_LOCATION=${RING_S3C_BACKEND_SOURCE_LOCATION} \
96-
--env=RING_S3C_INGESTION_SRC_BUCKET_NAME=${RING_S3C_INGESTION_SRC_BUCKET_NAME} \
97-
--env=RING_S3C_BACKEND_SOURCE_NON_VERSIONED_LOCATION=${RING_S3C_BACKEND_SOURCE_NON_VERSIONED_LOCATION} \
98-
--env=RING_S3C_INGESTION_SRC_NON_VERSIONED_BUCKET_NAME=${RING_S3C_INGESTION_SRC_NON_VERSIONED_BUCKET_NAME} \
99-
--env=RING_S3C_INGESTION_NON_VERSIONED_OBJECT_COUNT_PER_TYPE=${RING_S3C_INGESTION_NON_VERSIONED_OBJECT_COUNT_PER_TYPE} \
100-
--env=AWS_ACCESS_KEY=${AWS_ACCESS_KEY} \
101-
--env=AWS_SECRET_KEY=${AWS_SECRET_KEY} \
102-
--env=AWS_ENDPOINT=${AWS_ENDPOINT} \
103-
--env=AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
104-
--env=AWS_REPLICATION_FAIL_CTST_BUCKET_NAME=${AWS_REPLICATION_FAIL_CTST_BUCKET_NAME} \
105-
--env=AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
106-
--env=AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
107-
--env=AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \
108-
--env=AZURE_ACCOUNT_NAME=${AZURE_ACCOUNT_NAME} \
109-
--env=AZURE_SECRET_KEY=${AZURE_SECRET_KEY} \
110-
--env=AZURE_CRR_BUCKET_NAME=${AZURE_CRR_BUCKET_NAME} \
111-
--env=AZURE_ARCHIVE_BUCKET_NAME=${AZURE_ARCHIVE_BUCKET_NAME} \
112-
--env=AZURE_ARCHIVE_BUCKET_NAME_2=${AZURE_ARCHIVE_BUCKET_NAME_2} \
113-
--env=AZURE_ARCHIVE_QUEUE_NAME=${AZURE_ARCHIVE_QUEUE_NAME} \
114-
--env=CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \
115-
--env=CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \
116-
--env=CRR_SOURCE_ACCOUNT_NAME=${CRR_SOURCE_ACCOUNT_NAME} \
117-
--env=CRR_DESTINATION_ACCOUNT_NAME=${CRR_DESTINATION_ACCOUNT_NAME} \
118-
--env=CRR_ROLE_NAME=${CRR_ROLE_NAME} \
119-
--env=DEPLOY_CRR_LOCATIONS=${DEPLOY_CRR_LOCATIONS} \
120-
--command -- python3 configuration.py
66+
# Run configuration.py directly
67+
ZENKO_TESTS_DIR="$DIR/../../../tests/zenko_tests"
68+
pip3 install -r "$ZENKO_TESTS_DIR/requirements.txt"
69+
70+
cd "$ZENKO_TESTS_DIR"
71+
72+
# Generate e2e-config.yaml from template (uses original RING_S3C_ENDPOINT with port 8000 for in-cluster Zenko config)
73+
envsubst < e2e-config.yaml.template > e2e-config.yaml
74+
75+
# Override RING_S3C_ENDPOINT for out-of-cluster access (via ingress on port 80)
76+
RING_S3C_ENDPOINT_LOCAL="${RING_S3C_ENDPOINT}"
77+
if kubectl get namespace metadata &>/dev/null; then
78+
RING_S3C_ENDPOINT_LOCAL="http://s3c.local"
79+
fi
80+
TOKEN=${TOKEN} \
81+
UUID=${UUID} \
82+
MANAGEMENT_ENDPOINT=${MANAGEMENT_ENDPOINT} \
83+
IAM_ENDPOINT=${IAM_ENDPOINT} \
84+
STS_ENDPOINT=${STS_ENDPOINT} \
85+
NAMESPACE=${NAMESPACE} \
86+
VERIFY_CERTIFICATES=false \
87+
ENABLE_RING_TESTS=${ENABLE_RING_TESTS} \
88+
RING_S3C_ACCESS_KEY=${RING_S3C_ACCESS_KEY} \
89+
RING_S3C_SECRET_KEY=${RING_S3C_SECRET_KEY} \
90+
RING_S3C_ENDPOINT=${RING_S3C_ENDPOINT_LOCAL} \
91+
RING_S3C_BACKEND_SOURCE_LOCATION=${RING_S3C_BACKEND_SOURCE_LOCATION} \
92+
RING_S3C_INGESTION_SRC_BUCKET_NAME=${RING_S3C_INGESTION_SRC_BUCKET_NAME} \
93+
RING_S3C_BACKEND_SOURCE_NON_VERSIONED_LOCATION=${RING_S3C_BACKEND_SOURCE_NON_VERSIONED_LOCATION} \
94+
RING_S3C_INGESTION_SRC_NON_VERSIONED_BUCKET_NAME=${RING_S3C_INGESTION_SRC_NON_VERSIONED_BUCKET_NAME} \
95+
RING_S3C_INGESTION_NON_VERSIONED_OBJECT_COUNT_PER_TYPE=${RING_S3C_INGESTION_NON_VERSIONED_OBJECT_COUNT_PER_TYPE} \
96+
AWS_ACCESS_KEY=${AWS_ACCESS_KEY} \
97+
AWS_SECRET_KEY=${AWS_SECRET_KEY} \
98+
AWS_ENDPOINT=${AWS_ENDPOINT} \
99+
AWS_FAIL_BUCKET_NAME=${AWS_FAIL_BUCKET_NAME} \
100+
AWS_REPLICATION_FAIL_CTST_BUCKET_NAME=${AWS_REPLICATION_FAIL_CTST_BUCKET_NAME} \
101+
AZURE_BACKEND_DESTINATION_LOCATION=${AZURE_BACKEND_DESTINATION_LOCATION} \
102+
AZURE_BACKEND_ENDPOINT=${AZURE_BACKEND_ENDPOINT} \
103+
AZURE_BACKEND_QUEUE_ENDPOINT=${AZURE_BACKEND_QUEUE_ENDPOINT} \
104+
AZURE_ACCOUNT_NAME=${AZURE_ACCOUNT_NAME} \
105+
AZURE_SECRET_KEY=${AZURE_SECRET_KEY} \
106+
AZURE_CRR_BUCKET_NAME=${AZURE_CRR_BUCKET_NAME} \
107+
AZURE_ARCHIVE_BUCKET_NAME=${AZURE_ARCHIVE_BUCKET_NAME} \
108+
AZURE_ARCHIVE_BUCKET_NAME_2=${AZURE_ARCHIVE_BUCKET_NAME_2} \
109+
AZURE_ARCHIVE_QUEUE_NAME=${AZURE_ARCHIVE_QUEUE_NAME} \
110+
CRR_SOURCE_LOCATION_NAME=${CRR_SOURCE_LOCATION_NAME} \
111+
CRR_DESTINATION_LOCATION_NAME=${CRR_DESTINATION_LOCATION_NAME} \
112+
CRR_SOURCE_ACCOUNT_NAME=${CRR_SOURCE_ACCOUNT_NAME} \
113+
CRR_DESTINATION_ACCOUNT_NAME=${CRR_DESTINATION_ACCOUNT_NAME} \
114+
CRR_ROLE_NAME=${CRR_ROLE_NAME} \
115+
DEPLOY_CRR_LOCATIONS=${DEPLOY_CRR_LOCATIONS} \
116+
python3 configuration.py
121117

122118
## wait for updates to trigger zenko upgrades
123119
sleep 10

0 commit comments

Comments
 (0)