Skip to content

Commit 750ffec

Browse files
dumb0002dumb0002
committed
latest
Signed-off-by: dumb0002 <Braulio.Dumba@ibm.com>
1 parent 497019c commit 750ffec

File tree

3 files changed

+93
-46
lines changed

3 files changed

+93
-46
lines changed

‎.github/workflows/ci-e2e-openshift.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -867,6 +867,32 @@ jobs:
867867
if: always()
868868
run: kubectl get all -n "$FMA_NAMESPACE"
869869

870+
- name: Run OpenShift E2E tests
871+
env:
872+
# E2E test configuration matching Makefile variables
873+
ENVIRONMENT: openshift
874+
POLICIES_ENABLED: "false"
875+
LIMIT: "600"
876+
# Test namespace where objects are deployed
877+
NAMESPACE: ${{ env.FMA_NAMESPACE }}
878+
# Test object names from previous step
879+
INST: ${{ steps.test-objects.outputs.instance }}
880+
ISC: ${{ steps.test-objects.outputs.isc }}
881+
LC: ${{ steps.test-objects.outputs.lc }}
882+
RS: ${{ steps.test-objects.outputs.rs }}
883+
run: |
884+
echo "Running E2E tests on OpenShift with configuration:"
885+
echo " ENVIRONMENT: $ENVIRONMENT"
886+
echo " POLICIES_ENABLED: $POLICIES_ENABLED"
887+
echo " LIMIT: $LIMIT"
888+
echo " NAMESPACE: $NAMESPACE"
889+
echo " FMA_RELEASE_NAME: $FMA_RELEASE_NAME"
890+
echo " INST: $INST"
891+
echo " ISC: $ISC"
892+
echo " LC: $LC"
893+
echo " RS: $RS"
894+
make test-e2e-full
895+
870896
- name: List event objects
871897
if: always()
872898
run: kubectl get events -n "$FMA_NAMESPACE" --sort-by='.lastTimestamp'

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ LIMIT ?= 600
145145
# Run full e2e test suite on OpenShift cluster
146146
# Supports KUBECONFIG or in-cluster authentication
147147
# Set POLICIES_ENABLED=true to enable CEL policy validation tests
148-
.PHONY: test-e2e-full
148+
.PHONY: ocp-e2e-full
149149
test-e2e-full: ## Run full e2e test suite on OpenShift
150150
@echo "Running full e2e test suite on $(ENVIRONMENT)..."
151151
@echo "KUBECONFIG: $(KUBECONFIG)"

‎test/e2e/ocp-test.sh‎

Lines changed: 66 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ function expect() {
2323
local start=$(date)
2424
local limit=${LIMIT:-600}
2525
while true; do
26-
kubectl get pods -L dual-pods.llm-d.ai/dual,dual-pods.llm-d.ai/sleeping
26+
kubectl get pods -n "$namespace" -L dual-pods.llm-d.ai/dual,dual-pods.llm-d.ai/sleeping
2727
if eval "$1"; then return; fi
2828
if (( elapsed > limit )); then
2929
echo "Did not become true (from $start to $(date)): $1" >&2
@@ -39,12 +39,33 @@ function expect() {
3939

4040
: Basic Launcher Pod Creation
4141

42-
objs=$(test/e2e/mkobjs.sh)
43-
isc=$(echo $objs | awk '{print $1}')
44-
lc=$(echo $objs | awk '{print $2}')
45-
rslb=$(echo $objs | awk '{print $3}')
46-
isc2=$(echo $objs | awk '{print $4}')
47-
instlb=${rslb#my-request-}
42+
# Use environment variables from workflow
43+
echo "Using test objects from environment variables:"
44+
echo " NAMESPACE: $NAMESPACE"
45+
echo " ISC: $ISC"
46+
echo " LC: $LC"
47+
echo " RS: $RS"
48+
echo " INST: $INST"
49+
50+
isc="$ISC"
51+
lc="$LC"
52+
rslb="$RS"
53+
instlb="$INST"
54+
namespace="$NAMESPACE"
55+
56+
# Verify required environment variables are set
57+
if [ -z "$namespace" ] || [ -z "$isc" ] || [ -z "$lc" ] || [ -z "$rslb" ] || [ -z "$instlb" ]; then
58+
echo "ERROR: Required environment variables not set!" >&2
59+
echo " NAMESPACE=$NAMESPACE" >&2
60+
echo " ISC=$ISC" >&2
61+
echo " LC=$LC" >&2
62+
echo " RS=$RS" >&2
63+
echo " INST=$INST" >&2
64+
exit 1
65+
fi
66+
67+
# Note: isc2 not provided by workflow, set to empty for tests that need it
68+
isc2=""
4869

4970

5071
: Test CEL policy verification if enabled
@@ -60,116 +81,116 @@ fi
6081
: Instance Wake-up Fast Path
6182

6283
# Scale requester to 0 (instance should sleep in launcher)
63-
kubectl scale rs $rslb --replicas=0
84+
kubectl scale rs $rslb --replicas=0 -n "$namespace"
6485

65-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
86+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
6687

6788
# Launcher should remain
68-
kubectl get pod $launcherlb
89+
kubectl get pod $launcherlb -n "$namespace"
6990

7091
# Verify launcher is unbound (no dual label pointing to requester)
71-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
92+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
7293

7394
# Scale back up (should reuse same launcher and wake sleeping instance)
74-
kubectl scale rs $rslb --replicas=1
95+
kubectl scale rs $rslb --replicas=1 -n "$namespace"
7596

76-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
97+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
7798

78-
reqlb2=$(kubectl get pods -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
99+
reqlb2=$(kubectl get pods -n "$namespace" -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
79100

80101
# Should still be using the same launcher pod
81-
launcherlb2=$(kubectl get pods -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
102+
launcherlb2=$(kubectl get pods -n "$namespace" -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
82103
[ "$launcherlb2" == "$launcherlb" ]
83104

84105
# Verify new requester is bound to same launcher
85-
expect '[ "$(kubectl get pod $reqlb2 -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
106+
expect '[ "$(kubectl get pod $reqlb2 -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
86107

87108
# Verify launcher is bound to new requester
88-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb2" ]'
109+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb2" ]'
89110

90111
# Wait for requester to be ready (launcher should already be ready)
91112
date
92-
kubectl wait --for condition=Ready pod/$reqlb2 --timeout=30s
93-
kubectl wait --for condition=Ready pod/$launcherlb --timeout=5s
113+
kubectl wait --for condition=Ready pod/$reqlb2 -n "$namespace" --timeout=30s
114+
kubectl wait --for condition=Ready pod/$launcherlb -n "$namespace" --timeout=5s
94115

95116
cheer Successful instance wake-up fast path
96117

97118
: Multiple Instances Share One Launcher
98119

99120
# Scale requester to 0 again
100-
kubectl scale rs $rslb --replicas=0
121+
kubectl scale rs $rslb --replicas=0 -n "$namespace"
101122

102-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
123+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
103124

104125
# Launcher should remain
105-
kubectl get pod $launcherlb
126+
kubectl get pod $launcherlb -n "$namespace"
106127

107128
# Verify launcher is unbound
108-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
129+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
109130

110131
# Patch ReplicaSet to use isc2 instead of isc
111-
kubectl patch rs $rslb -p='{"spec":{"template":{"metadata":{"annotations":{"dual-pods.llm-d.ai/inference-server-config":"'$isc2'"}}}}}'
132+
kubectl patch rs $rslb -n "$namespace" -p='{"spec":{"template":{"metadata":{"annotations":{"dual-pods.llm-d.ai/inference-server-config":"'$isc2'"}}}}}'
112133

113134
# Scale back up (should reuse same launcher and create 2nd instance)
114-
kubectl scale rs $rslb --replicas=1
135+
kubectl scale rs $rslb --replicas=1 -n "$namespace"
115136

116-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
137+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
117138

118-
reqlb3=$(kubectl get pods -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
139+
reqlb3=$(kubectl get pods -n "$namespace" -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
119140

120141
# Should still be using the same launcher pod
121-
launcherlb3=$(kubectl get pods -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
142+
launcherlb3=$(kubectl get pods -n "$namespace" -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
122143
[ "$launcherlb3" == "$launcherlb" ]
123144

124145
# Verify new requester is bound to same launcher
125-
expect '[ "$(kubectl get pod $reqlb3 -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
146+
expect '[ "$(kubectl get pod $reqlb3 -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
126147

127148
# Verify launcher is bound to new requester
128-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb3" ]'
149+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb3" ]'
129150

130151
# Wait for requester to be ready (launcher should already be ready)
131152
date
132-
kubectl wait --for condition=Ready pod/$reqlb3 --timeout=120s
133-
kubectl wait --for condition=Ready pod/$launcherlb --timeout=5s
153+
kubectl wait --for condition=Ready pod/$reqlb3 -n "$namespace" --timeout=120s
154+
kubectl wait --for condition=Ready pod/$launcherlb -n "$namespace" --timeout=5s
134155

135156
cheer Successful multiple instances sharing one launcher
136157

137158
: Switch Instances In One Launcher
138159

139160
# Scale requester to 0 again
140-
kubectl scale rs $rslb --replicas=0
161+
kubectl scale rs $rslb --replicas=0 -n "$namespace"
141162

142-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
163+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | wc -l | grep -w 0"
143164

144165
# Launcher should remain
145-
kubectl get pod $launcherlb
166+
kubectl get pod $launcherlb -n "$namespace"
146167

147168
# Verify launcher is unbound
148-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
169+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "" ]'
149170

150171
# Patch ReplicaSet back to use original isc
151-
kubectl patch rs $rslb -p='{"spec":{"template":{"metadata":{"annotations":{"dual-pods.llm-d.ai/inference-server-config":"'$isc'"}}}}}'
172+
kubectl patch rs $rslb -n "$namespace" -p='{"spec":{"template":{"metadata":{"annotations":{"dual-pods.llm-d.ai/inference-server-config":"'$isc'"}}}}}'
152173

153174
# Scale back up (should reuse same launcher and wake first instance)
154-
kubectl scale rs $rslb --replicas=1
175+
kubectl scale rs $rslb --replicas=1 -n "$namespace"
155176

156-
expect "kubectl get pods -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
177+
expect "kubectl get pods -n '$namespace' -o name -l app=dp-example,instance=$instlb | grep -c '^pod/' | grep -w 1"
157178

158-
reqlb4=$(kubectl get pods -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
179+
reqlb4=$(kubectl get pods -n "$namespace" -o name -l app=dp-example,instance=$instlb | sed s%pod/%%)
159180

160181
# Should still be using the same launcher pod
161-
launcherlb4=$(kubectl get pods -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
182+
launcherlb4=$(kubectl get pods -n "$namespace" -o name -l dual-pods.llm-d.ai/launcher-config-name=$lc | sed s%pod/%%)
162183
[ "$launcherlb4" == "$launcherlb" ]
163184

164185
# Verify new requester is bound to same launcher
165-
expect '[ "$(kubectl get pod $reqlb4 -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
186+
expect '[ "$(kubectl get pod $reqlb4 -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$launcherlb" ]'
166187

167188
# Verify launcher is bound to new requester
168-
expect '[ "$(kubectl get pod $launcherlb -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb4" ]'
189+
expect '[ "$(kubectl get pod $launcherlb -n '"$namespace"' -o jsonpath={.metadata.labels.dual-pods\\.llm-d\\.ai/dual})" == "$reqlb4" ]'
169190

170191
# Wait for requester to be ready (launcher should already be ready)
171192
date
172-
kubectl wait --for condition=Ready pod/$reqlb4 --timeout=120s
173-
kubectl wait --for condition=Ready pod/$launcherlb --timeout=5s
193+
kubectl wait --for condition=Ready pod/$reqlb4 -n "$namespace" --timeout=120s
194+
kubectl wait --for condition=Ready pod/$launcherlb -n "$namespace" --timeout=5s
174195

175196
cheer Successful switching instances in one launcher

0 commit comments

Comments
 (0)