Skip to content

Commit b93a1c8

Browse files
fabianvffabianvf
authored
👻 Add more verbose output and longer waits during llm-proxy test (#535)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Tests** * Improved Keycloak readiness checks and admin-user readiness polling with clearer timeouts and warnings. * Hardened admin-user configuration tests to handle missing secrets and non-ideal responses gracefully. * Added retry logic and robust token extraction for authentication tests with clearer status logs. * Made LLM proxy tests conditional on valid tokens, added security checks for invalid tokens, and improved error reporting. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: Fabian von Feilitzsch <fabian@fabianism.us>
1 parent 6b526c5 commit b93a1c8

File tree

1 file changed

+155
-45
lines changed

1 file changed

+155
-45
lines changed

‎test/e2e/llm-proxy/test-llm-proxy.sh‎

Lines changed: 155 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,57 @@ wait_for_deployment() {
3535
wait_for_deployment tackle-hub
3636
wait_for_deployment llm-proxy
3737
wait_for_deployment llemulator
38+
wait_for_deployment tackle-keycloak-sso
39+
40+
# Wait for Keycloak to be fully ready (not just the pod, but the service responding)
41+
echo -n "Waiting for Keycloak to be ready..."
42+
KC_READY=false
43+
for i in $(seq 1 30); do
44+
KC_STATUS=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -o /dev/null -w "%{http_code}" \
45+
http://tackle-keycloak-sso:8080/auth/realms/tackle/.well-known/openid-configuration 2>/dev/null || echo "000")
46+
if [ "$KC_STATUS" = "200" ]; then
47+
echo " ready"
48+
KC_READY=true
49+
break
50+
fi
51+
echo -n "."
52+
sleep 5
53+
done
54+
if [ "$KC_READY" != true ]; then
55+
echo " timeout (Keycloak may not be fully ready)"
56+
fi
57+
58+
# Wait for admin user to exist in tackle realm (created by keycloak job)
59+
echo -n "Waiting for admin user in tackle realm..."
60+
ADMIN_EXISTS=false
61+
ADMIN_SECRET=$(kubectl get secret tackle-keycloak-sso -n $NAMESPACE -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || true)
62+
for i in $(seq 1 30); do
63+
# Get admin token
64+
ADMIN_TOKEN_RESP=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X POST \
65+
http://tackle-keycloak-sso:8080/auth/realms/master/protocol/openid-connect/token \
66+
-H "Content-Type: application/x-www-form-urlencoded" \
67+
-d "grant_type=password&client_id=admin-cli&username=admin&password=$ADMIN_SECRET" 2>/dev/null || echo "{}")
68+
69+
if echo "$ADMIN_TOKEN_RESP" | grep -q "access_token"; then
70+
TEMP_TOKEN=$(echo "$ADMIN_TOKEN_RESP" | jq -r '.access_token' 2>/dev/null || true)
71+
if [ -n "$TEMP_TOKEN" ]; then
72+
# Check for admin user in tackle realm
73+
USERS=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s \
74+
"http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users?username=admin" \
75+
-H "Authorization: Bearer $TEMP_TOKEN" 2>/dev/null || echo "[]")
76+
if echo "$USERS" | grep -q '"username":"admin"'; then
77+
echo " found"
78+
ADMIN_EXISTS=true
79+
break
80+
fi
81+
fi
82+
fi
83+
echo -n "."
84+
sleep 5
85+
done
86+
if [ "$ADMIN_EXISTS" != true ]; then
87+
echo " timeout (admin user may not exist yet)"
88+
fi
3889

3990
# Get hub URL
4091
HUB_URL=""
@@ -58,60 +109,119 @@ echo "Hub URL: $HUB_URL"
58109

59110
# Clear password change requirement for admin user
60111
echo "Configuring authentication..."
61-
ADMIN_SECRET=$(kubectl get secret tackle-keycloak-sso -n $NAMESPACE -o jsonpath='{.data.password}' | base64 -d)
62-
63-
# Get admin token from Keycloak
64-
ADMIN_TOKEN_RESPONSE=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X POST \
65-
http://tackle-keycloak-sso:8080/auth/realms/master/protocol/openid-connect/token \
66-
-H "Content-Type: application/x-www-form-urlencoded" \
67-
-d "grant_type=password" \
68-
-d "client_id=admin-cli" \
69-
-d "username=admin" \
70-
-d "password=$ADMIN_SECRET" 2>/dev/null || echo "{}")
71-
72-
ADMIN_TOKEN=$(echo "$ADMIN_TOKEN_RESPONSE" | jq -r '.access_token // empty')
73-
74-
if [ -n "$ADMIN_TOKEN" ] && [ "$ADMIN_TOKEN" != "null" ]; then
75-
# Get admin user ID in tackle realm
76-
ADMIN_USER_ID=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s \
77-
http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users \
78-
-H "Authorization: Bearer $ADMIN_TOKEN" | jq -r '.[] | select(.username=="admin") | .id // empty')
79-
80-
if [ -n "$ADMIN_USER_ID" ]; then
81-
# Clear required actions and reset password
82-
kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X PUT \
83-
"http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users/$ADMIN_USER_ID" \
84-
-H "Authorization: Bearer $ADMIN_TOKEN" \
85-
-H "Content-Type: application/json" \
86-
-d '{"requiredActions": []}' &>/dev/null
112+
# ADMIN_SECRET was already retrieved in the wait loop above
87113

88-
kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X PUT \
89-
"http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users/$ADMIN_USER_ID/reset-password" \
90-
-H "Authorization: Bearer $ADMIN_TOKEN" \
91-
-H "Content-Type: application/json" \
92-
-d '{"type": "password", "value": "Passw0rd!", "temporary": false}' &>/dev/null
114+
if [ -z "$ADMIN_SECRET" ]; then
115+
echo " Warning: Could not get Keycloak admin secret, skipping admin user configuration"
116+
else
117+
# Always try to configure admin user (even if wait timed out, user might exist now)
118+
# Get admin token from Keycloak
119+
ADMIN_TOKEN_RESPONSE=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X POST \
120+
http://tackle-keycloak-sso:8080/auth/realms/master/protocol/openid-connect/token \
121+
-H "Content-Type: application/x-www-form-urlencoded" \
122+
-d "grant_type=password" \
123+
-d "client_id=admin-cli" \
124+
-d "username=admin" \
125+
-d "password=$ADMIN_SECRET" 2>/dev/null || echo "{}")
126+
127+
# Safely extract token (handle non-JSON responses)
128+
if echo "$ADMIN_TOKEN_RESPONSE" | grep -q "access_token"; then
129+
ADMIN_TOKEN=$(echo "$ADMIN_TOKEN_RESPONSE" | jq -r '.access_token // empty' 2>/dev/null || true)
130+
else
131+
ADMIN_TOKEN=""
132+
echo " Warning: Could not get Keycloak admin token (response: $(echo "$ADMIN_TOKEN_RESPONSE" | head -c 200))"
133+
fi
134+
135+
if [ -n "$ADMIN_TOKEN" ] && [ "$ADMIN_TOKEN" != "null" ]; then
136+
echo " Got Keycloak admin token, configuring admin user..."
137+
# Get admin user ID in tackle realm
138+
USERS_RESPONSE=$(kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s \
139+
http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users \
140+
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null || echo "[]")
141+
142+
if echo "$USERS_RESPONSE" | grep -q "username"; then
143+
ADMIN_USER_ID=$(echo "$USERS_RESPONSE" | jq -r '.[] | select(.username=="admin") | .id // empty' 2>/dev/null || true)
144+
else
145+
ADMIN_USER_ID=""
146+
fi
147+
148+
if [ -n "$ADMIN_USER_ID" ]; then
149+
echo " Found admin user ID: $ADMIN_USER_ID"
150+
# Clear required actions and reset password
151+
kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X PUT \
152+
"http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users/$ADMIN_USER_ID" \
153+
-H "Authorization: Bearer $ADMIN_TOKEN" \
154+
-H "Content-Type: application/json" \
155+
-d '{"requiredActions": []}' &>/dev/null || true
156+
157+
kubectl exec -n $NAMESPACE deployment/tackle-hub -- curl -s -X PUT \
158+
"http://tackle-keycloak-sso:8080/auth/admin/realms/tackle/users/$ADMIN_USER_ID/reset-password" \
159+
-H "Authorization: Bearer $ADMIN_TOKEN" \
160+
-H "Content-Type: application/json" \
161+
-d '{"type": "password", "value": "Passw0rd!", "temporary": false}' &>/dev/null || true
162+
echo " Admin user configured"
163+
else
164+
echo " Warning: Could not find admin user in tackle realm"
165+
fi
166+
else
167+
echo " Warning: Skipping admin user configuration (no admin token)"
93168
fi
94169
fi
95170

96-
# Test hub authentication
171+
# Test hub authentication (with retries since Keycloak may need time)
97172
echo "Testing hub authentication..."
98-
HUB_AUTH_RESPONSE=$(curl -s -X POST \
99-
$HUB_URL/hub/auth/login \
100-
-H "Content-Type: application/json" \
101-
-d '{"user": "admin", "password": "Passw0rd!"}' \
102-
-D /tmp/auth_headers.txt 2>/dev/null)
173+
ACCESS_TOKEN=""
174+
for attempt in 1 2 3; do
175+
echo " Attempt $attempt/3..."
176+
HUB_AUTH_RESPONSE=$(curl -s -X POST \
177+
$HUB_URL/hub/auth/login \
178+
-H "Content-Type: application/json" \
179+
-d '{"user": "admin", "password": "Passw0rd!"}' \
180+
--connect-timeout 10 \
181+
--max-time 30 \
182+
-D /tmp/auth_headers.txt -w "\n%{http_code}" 2>/dev/null || echo "CURL_FAILED")
103183

104-
# Extract token
105-
ACCESS_TOKEN=$(grep -i "authorization" /tmp/auth_headers.txt 2>/dev/null | sed 's/.*Bearer //' | tr -d '\r\n')
106-
if [ -z "$ACCESS_TOKEN" ]; then
107-
ACCESS_TOKEN=$(echo "$HUB_AUTH_RESPONSE" | jq -r '.token // .access_token // empty' 2>/dev/null)
108-
fi
184+
# Extract HTTP status code (last line)
185+
HTTP_STATUS=$(echo "$HUB_AUTH_RESPONSE" | tail -1)
186+
# Remove status code from response body
187+
HUB_AUTH_RESPONSE=$(echo "$HUB_AUTH_RESPONSE" | sed '$d')
188+
189+
echo " Hub auth response status: $HTTP_STATUS"
190+
191+
# Check if curl failed or got non-2xx response
192+
if [ "$HTTP_STATUS" = "CURL_FAILED" ]; then
193+
echo " Warning: curl failed to connect to hub"
194+
elif [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 400 ] 2>/dev/null; then
195+
echo " Warning: Hub authentication failed with HTTP $HTTP_STATUS"
196+
echo " Response preview: $(echo "$HUB_AUTH_RESPONSE" | head -c 200)"
197+
else
198+
# Extract token from headers first
199+
ACCESS_TOKEN=$(grep -i "authorization" /tmp/auth_headers.txt 2>/dev/null | sed 's/.*Bearer //' | tr -d '\r\n' || true)
200+
201+
# If not in headers, try to parse from JSON body
202+
if [ -z "$ACCESS_TOKEN" ]; then
203+
# Only parse if response looks like JSON
204+
if echo "$HUB_AUTH_RESPONSE" | grep -q "^{"; then
205+
ACCESS_TOKEN=$(echo "$HUB_AUTH_RESPONSE" | jq -r '.token // .access_token // empty' 2>/dev/null || true)
206+
fi
207+
fi
208+
209+
if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
210+
echo "Authentication successful"
211+
break
212+
fi
213+
fi
214+
215+
if [ $attempt -lt 3 ]; then
216+
echo " Retrying in 5 seconds..."
217+
sleep 5
218+
fi
219+
done
109220

110221
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" == "null" ]; then
111-
echo "ERROR: Failed to get authentication token"
222+
echo "ERROR: Failed to get authentication token after 3 attempts"
223+
echo " Last response headers: $(cat /tmp/auth_headers.txt 2>/dev/null | head -10)"
112224
TEST_FAILED=true
113-
else
114-
echo "Authentication successful"
115225
fi
116226

117227
# Test LLM proxy with all configured responses

0 commit comments

Comments
 (0)