Skip to content

Commit 6a47e37

Browse files
SB159SB159
authored
test: smoke test suite, scripts, and readme (requirements + setup) (#165)
* WIP: e2e smoke tests + scripts * docs(e2e): add README with required exports and flow * docs(e2e): add venv setup and requirements
1 parent ce21e65 commit 6a47e37

File tree

9 files changed

+517
-0
lines changed

9 files changed

+517
-0
lines changed

‎testing/e2e/.gitignore‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Python caches / local env
2+
__pycache__/
3+
.pytest_cache/
4+
.venv/
5+
.env
6+
7+
# e2e run artifacts
8+
reports/
9+
*.log
10+
11+
# editor/OS cruft
12+
.vscode/
13+
.DS_Store

‎testing/e2e/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# e2e Smoke Tests
2+
3+
## Setup (one-time per workspace)
4+
5+
**Prereqs:** Python 3.11+, `oc`, `kubectl`, `kustomize`, `jq` (and `yq` optional).
6+
7+
```bash
8+
# create & activate a virtualenv
9+
python3.11 -m venv .venv
10+
source .venv/bin/activate # Windows PowerShell: .\.venv\Scripts\Activate.ps1
11+
12+
# install Python deps used by the tests
13+
pip install --upgrade pip
14+
pip install -r testing/e2e/requirements.txt
15+
16+
## What Prow needs to provide (exports)
17+
- `CLUSTER_DOMAIN` – from cluster (e.g., `oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}'`)
18+
- `HOST` – maas.${CLUSTER_DOMAIN}
19+
- `MAAS_API_BASE_URL` – `https://${HOST}/maas-api` (or `http://` if TLS isn’t ready)
20+
- `MODEL_NAME` – gateway model id (e.g., `facebook/opt-125m`)
21+
- `ISVC_NAME` (optional) – CR name without slashes (e.g., `facebook-opt-125m-cpu`) if you deploy a sample
22+
23+
## Typical run
24+
```bash
25+
export CLUSTER_DOMAIN="$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}')"
26+
export HOST="maas.${CLUSTER_DOMAIN}"
27+
export MAAS_API_BASE_URL="https://${HOST}/maas-api"
28+
export MODEL_NAME="facebook/opt-125m"
29+
# optional if you deploy a sample:
30+
# export ISVC_NAME="facebook-opt-125m-cpu"
31+
32+
# Deploy + smoke:
33+
bash testing/e2e/run-model-and-smoke.sh
34+
# Or only smoke (if model is already there):
35+
bash testing/e2e/smoke.sh

‎testing/e2e/bootstrap.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
trap 'echo "[bootstrap] ERROR line $LINENO: $BASH_COMMAND" >&2' ERR
4+
5+
# Repo/E2E dirs
6+
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
7+
E2E_DIR="${REPO_ROOT}/testing/e2e"
8+
9+
# Defaults (overridable via env)
10+
NS="${NS:-llm}"
11+
MODEL_PATH="${MODEL_PATH:-docs/samples/models/facebook-opt-125m-cpu}"
12+
GATEWAY_NAME="${GATEWAY_NAME:-maas-default-gateway}"
13+
GATEWAY_NS="${GATEWAY_NS:-openshift-ingress}"
14+
WRITE_ENV="${WRITE_ENV:-true}" # write testing/e2e/.env
15+
SKIP_DEPLOY="${SKIP_DEPLOY:-true}" # default true (don’t redeploy model unless you want to)
16+
17+
echo "[bootstrap] oc whoami: $(oc whoami || true)"
18+
echo "[bootstrap] NS=${NS} MODEL_PATH=${MODEL_PATH} SKIP_DEPLOY=${SKIP_DEPLOY}"
19+
20+
command -v oc >/dev/null 2>&1 || { echo "oc missing"; exit 1; }
21+
command -v jq >/dev/null 2>&1 || { echo "jq missing"; exit 1; }
22+
command -v kustomize >/dev/null 2>&1 || { echo "kustomize missing"; exit 1; }
23+
24+
# ---- Detect model CR name from kustomize
25+
if command -v yq >/dev/null 2>&1; then
26+
DEDUCED_CR="$(kustomize build "${MODEL_PATH}" | yq -r 'select(.kind=="LLMInferenceService") | .metadata.name' | head -n1)"
27+
else
28+
DEDUCED_CR="$(kustomize build "${MODEL_PATH}" | awk '/^kind: LLMInferenceService$/{f=1} f&&/^ name:/{print $2; exit}')"
29+
fi
30+
if [[ -z "${DEDUCED_CR:-}" ]]; then
31+
echo "[bootstrap] Could not detect LLMInferenceService name from ${MODEL_PATH}" >&2
32+
exit 1
33+
fi
34+
export MODEL_NAME="${MODEL_NAME:-$DEDUCED_CR}"
35+
echo "[bootstrap] Using kind=llminferenceservice ns=${NS} (${MODEL_PATH##*/})"
36+
echo "[bootstrap] Model CR name: ${MODEL_NAME}"
37+
38+
# ---- (Optional) deploy/redeploy the model
39+
if [[ "${SKIP_DEPLOY}" != "true" ]]; then
40+
oc get ns "${NS}" >/dev/null 2>&1 || oc create ns "${NS}"
41+
echo "[bootstrap] Applying from: ${MODEL_PATH}/"
42+
kustomize build "${MODEL_PATH}" | kubectl apply -f -
43+
echo "[bootstrap] Waiting for llminferenceservice/${MODEL_NAME} to be Ready (timeout 15m)…"
44+
oc -n "${NS}" wait --for=condition=Ready "llminferenceservice/${MODEL_NAME}" --timeout=15m
45+
else
46+
echo "[bootstrap] Skipping model deployment (SKIP_DEPLOY=${SKIP_DEPLOY})"
47+
fi
48+
49+
# ---- Discover gateway host and MaaS API URL
50+
HOST="${HOST:-}"
51+
if [[ -z "${HOST}" ]]; then
52+
HOST="$(oc -n "${GATEWAY_NS}" get gateway "${GATEWAY_NAME}" -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)"
53+
fi
54+
if [[ -z "${HOST}" ]]; then
55+
# Fallback to cluster apps domain
56+
APPS="$(oc get ingresses.config/cluster -o jsonpath='{.spec.domain}' 2>/dev/null || oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}' 2>/dev/null || true)"
57+
HOST="gateway.${APPS}"
58+
fi
59+
if [[ -z "${HOST}" ]]; then
60+
echo "[bootstrap] ERROR: could not determine HOST" >&2
61+
exit 1
62+
fi
63+
64+
# Prefer https if healthz responds; otherwise http
65+
SCHEME="https"
66+
if ! curl -skS -m 5 "${SCHEME}://${HOST}/maas-api/healthz" -o /dev/null ; then
67+
SCHEME="http"
68+
fi
69+
70+
export HOST
71+
export MAAS_API_BASE_URL="${SCHEME}://${HOST}/maas-api"
72+
73+
# Try to discover model base URL via catalog (nice-to-have)
74+
FREE_OC_TOKEN="$(oc whoami -t || true)"
75+
MODEL_URL_DISC=""
76+
if [[ -n "${FREE_OC_TOKEN}" ]]; then
77+
TOKEN_RESPONSE="$(curl -sSk -H "Authorization: Bearer ${FREE_OC_TOKEN}" -H "Content-Type: application/json" -X POST -d '{"expiration":"10m"}' "${MAAS_API_BASE_URL}/v1/tokens" || true)"
78+
TOKEN="$(echo "${TOKEN_RESPONSE}" | jq -r .token 2>/dev/null || true)"
79+
if [[ -n "${TOKEN}" && "${TOKEN}" != "null" ]]; then
80+
MODELS_JSON="$(curl -sSk -H "Authorization: Bearer ${TOKEN}" "${MAAS_API_BASE_URL}/v1/models" || true)"
81+
MODEL_URL_DISC="$(echo "${MODELS_JSON}" | jq -r '(.data // .models // [])[0]?.url // empty' 2>/dev/null || true)"
82+
fi
83+
fi
84+
85+
# Compose model URL if catalog didn’t give us one
86+
if [[ -z "${MODEL_URL_DISC}" ]]; then
87+
MODEL_URL_DISC="${SCHEME}://${HOST}/llm/${MODEL_NAME}"
88+
fi
89+
MODEL_URL="${MODEL_URL_DISC%/}/v1"
90+
91+
echo "[bootstrap] MAAS_API_BASE_URL=${MAAS_API_BASE_URL}"
92+
echo "[bootstrap] MODEL_URL=${MODEL_URL}"
93+
94+
# ---- Write .env for convenience
95+
if [[ "${WRITE_ENV}" == "true" ]]; then
96+
mkdir -p "${E2E_DIR}"
97+
cat > "${E2E_DIR}/.env" <<EOF
98+
export HOST="${HOST}"
99+
export MAAS_API_BASE_URL="${MAAS_API_BASE_URL}"
100+
export FREE_OC_TOKEN="$(oc whoami -t || true)"
101+
export MODEL_NAME="${MODEL_NAME}"
102+
export MODEL_URL="${MODEL_URL}"
103+
export ROUTER_MODE="gw" # informational only
104+
EOF
105+
echo "[bootstrap] wrote ${E2E_DIR}/.env"
106+
fi
107+
108+
echo "[bootstrap] Done."

‎testing/e2e/requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
pytest>=8.0,<9
2+
pytest-html>=4.1,<5
3+
pytest-metadata>=3.1,<4
4+
requests>=2.32,<3

‎testing/e2e/run-model-and-smoke.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# -------- Config (overridable) --------
5+
NS="${NS:-llm}"
6+
MODEL_PATH="${MODEL_PATH:-docs/samples/models/facebook-opt-125m-cpu}"
7+
ARTIFACT_DIR="${ARTIFACT_DIR:-testing/e2e/reports}"
8+
9+
# -------- Inputs we require --------
10+
: "${MODEL_NAME:?Export MODEL_NAME (LLMInferenceService name)}"
11+
12+
# One of these must be provided (we'll derive HOST/API if needed)
13+
CLUSTER_DOMAIN="${CLUSTER_DOMAIN:-}"
14+
HOST="${HOST:-}"
15+
MAAS_API_BASE_URL="${MAAS_API_BASE_URL:-}"
16+
17+
echo "[e2e] NS=${NS}"
18+
echo "[e2e] MODEL_NAME=${MODEL_NAME}"
19+
echo "[e2e] MODEL_PATH=${MODEL_PATH}"
20+
21+
command -v oc >/dev/null || { echo "oc missing"; exit 1; }
22+
command -v kustomize >/dev/null || { echo "kustomize missing"; exit 1; }
23+
24+
# -------- Deploy model --------
25+
oc get ns "${NS}" >/dev/null 2>&1 || oc create ns "${NS}"
26+
echo "[e2e] Applying ${MODEL_PATH} to ns ${NS}"
27+
kustomize build "${MODEL_PATH}" | kubectl -n "${NS}" apply -f -
28+
echo "[e2e] Waiting for LLMInferenceService/${MODEL_NAME} Ready…"
29+
ISVC_NAME="${ISVC_NAME:-}"
30+
if [[ -z "${ISVC_NAME}" ]]; then
31+
if command -v yq >/dev/null 2>&1; then
32+
ISVC_NAME="$(kustomize build "${MODEL_PATH}" | yq -r 'select(.kind=="LLMInferenceService") | .metadata.name' | head -n1)"
33+
else
34+
ISVC_NAME="$(kustomize build "${MODEL_PATH}" | awk '/^kind: LLMInferenceService$/{f=1} f&&/^ name:/{print $2; exit}')"
35+
fi
36+
fi
37+
oc -n "${NS}" wait --for=condition=Ready "llminferenceservice/${ISVC_NAME}" --timeout=15m
38+
39+
# -------- Work out API base URL (simple rules) --------
40+
if [[ -z "${MAAS_API_BASE_URL}" ]]; then
41+
if [[ -z "${HOST}" ]]; then
42+
if [[ -z "${CLUSTER_DOMAIN}" ]]; then
43+
echo "[e2e] ERROR: set MAAS_API_BASE_URL or HOST or CLUSTER_DOMAIN" >&2
44+
exit 2
45+
fi
46+
HOST="maas.${CLUSTER_DOMAIN}"
47+
fi
48+
SCHEME="https"
49+
if ! curl -skI -m 5 "${SCHEME}://${HOST}/maas-api/healthz" >/dev/null; then
50+
SCHEME="http"
51+
fi
52+
MAAS_API_BASE_URL="${SCHEME}://${HOST}/maas-api"
53+
fi
54+
55+
export HOST
56+
export MAAS_API_BASE_URL
57+
export MODEL_NAME
58+
59+
echo "[e2e] HOST=${HOST}"
60+
echo "[e2e] MAAS_API_BASE_URL=${MAAS_API_BASE_URL}"
61+
62+
# -------- Run smoke --------
63+
mkdir -p "${ARTIFACT_DIR}"
64+
echo "[e2e] Running smoke tests…"
65+
( cd testing/e2e && bash ./smoke.sh )
66+
67+
# Copy artifacts if a different dir was requested
68+
if [[ "testing/e2e/reports" != "${ARTIFACT_DIR}" ]]; then
69+
cp -r testing/e2e/reports/. "${ARTIFACT_DIR}/"
70+
fi
71+
echo "[e2e] Done. Reports in ${ARTIFACT_DIR}"

‎testing/e2e/smoke.sh‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
DIR="$(cd "$(dirname "$0")" && pwd)"
5+
export PYTHONPATH="${DIR}:${PYTHONPATH:-}"
6+
7+
# Inputs via env or auto-discovery
8+
HOST="${HOST:-}"
9+
MAAS_API_BASE_URL="${MAAS_API_BASE_URL:-}"
10+
MODEL_NAME="${MODEL_NAME:-}"
11+
12+
# If API base URL missing, derive from HOST, or discover HOST if needed
13+
if [[ -z "${MAAS_API_BASE_URL}" ]]; then
14+
if [[ -z "${HOST}" ]]; then
15+
GATEWAY_NAME="${GATEWAY_NAME:-maas-default-gateway}"
16+
GATEWAY_NS="${GATEWAY_NS:-openshift-ingress}"
17+
HOST="$(oc -n "${GATEWAY_NS}" get gateway "${GATEWAY_NAME}" -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)"
18+
if [[ -z "${HOST}" ]]; then
19+
APPS="$(oc get ingresses.config/cluster -o jsonpath='{.spec.domain}' 2>/dev/null \
20+
|| oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}' 2>/dev/null || true)"
21+
HOST="gateway.${APPS}"
22+
fi
23+
fi
24+
SCHEME="https"
25+
if ! curl -skS -m 5 "${SCHEME}://${HOST}/maas-api/healthz" -o /dev/null ; then
26+
SCHEME="http"
27+
fi
28+
MAAS_API_BASE_URL="${SCHEME}://${HOST}/maas-api"
29+
fi
30+
31+
echo "[smoke] MAAS_API_BASE_URL=${MAAS_API_BASE_URL}"
32+
if [[ -n "${MODEL_NAME}" ]]; then
33+
echo "[smoke] Using MODEL_NAME=${MODEL_NAME}"
34+
fi
35+
36+
# 1) Mint a MaaS token using your cluster token
37+
mkdir -p "${DIR}/reports"
38+
LOG="${DIR}/reports/smoke.log"
39+
: > "${LOG}"
40+
41+
FREE_OC_TOKEN="$(oc whoami -t || true)"
42+
TOKEN_RESPONSE="$(curl -skS \
43+
-H "Authorization: Bearer ${FREE_OC_TOKEN}" \
44+
-H "Content-Type: application/json" \
45+
-X POST \
46+
-d '{"expiration":"10m"}' \
47+
"${MAAS_API_BASE_URL}/v1/tokens" || true)"
48+
49+
TOKEN="$(echo "${TOKEN_RESPONSE}" | jq -r .token 2>/dev/null || true)"
50+
if [[ -z "${TOKEN}" || "${TOKEN}" == "null" ]]; then
51+
echo "[smoke] ERROR: could not mint MaaS token" | tee -a "${LOG}"
52+
echo "${TOKEN_RESPONSE}" | tee -a "${LOG}"
53+
exit 1
54+
fi
55+
export TOKEN
56+
57+
# Log a masked preview of the token to the log (not the console)
58+
echo "[token] minted: len=$((${#TOKEN})) head=${TOKEN:0:12}…tail=${TOKEN: -8}" >> "${LOG}"
59+
60+
# 2) Get models, derive URL/ID if catalog returns them
61+
MODELS_JSON="$(curl -skS -H "Authorization: Bearer ${TOKEN}" "${MAAS_API_BASE_URL}/v1/models" || true)"
62+
MODEL_URL="$(echo "${MODELS_JSON}" | jq -r '(.data // .models // [])[0]?.url // empty' 2>/dev/null || true)"
63+
MODEL_ID="$(echo "${MODELS_JSON}" | jq -r '(.data // .models // [])[0]?.id // empty' 2>/dev/null || true)"
64+
65+
# Fallbacks
66+
if [[ -z "${MODEL_ID}" || "${MODEL_ID}" == "null" ]]; then
67+
if [[ -z "${MODEL_NAME:-}" ]]; then
68+
echo "[smoke] ERROR: catalog did not return a model id and MODEL_NAME not set" | tee -a "${LOG}"
69+
exit 2
70+
fi
71+
MODEL_ID="${MODEL_NAME}"
72+
fi
73+
74+
if [[ -z "${MODEL_URL}" || "${MODEL_URL}" == "null" ]]; then
75+
_base="${MAAS_API_BASE_URL%/maas-api}"
76+
_base="${_base#https://}"; _base="${_base#http://}"
77+
MODEL_URL="https://${_base}/llm/${MODEL_ID}"
78+
fi
79+
80+
export MODEL_URL="${MODEL_URL%/}/v1"
81+
export MODEL_NAME="${MODEL_ID}"
82+
echo "[smoke] Using MODEL_URL=${MODEL_URL}" | tee -a "${LOG}"
83+
84+
# 3) Pytest outputs
85+
HTML="${DIR}/reports/smoke.html"
86+
XML="${DIR}/reports/smoke.xml"
87+
88+
PYTEST_ARGS=(
89+
-q
90+
--maxfail=1
91+
--disable-warnings
92+
"--junitxml=${XML}"
93+
# ⬇️ add these 3 so output shows up in the HTML:
94+
--html="${HTML}" --self-contained-html
95+
--capture=tee-sys # capture prints and also echo to console
96+
--show-capture=all # include captured output in the report
97+
--log-level=INFO # capture logging at INFO and above
98+
"${DIR}/tests/test_smoke.py"
99+
)
100+
101+
python -c 'import pytest_html' >/dev/null 2>&1 || echo "[smoke] WARNING: pytest-html not found (but we still passed --html)"
102+
103+
pytest "${PYTEST_ARGS[@]}"
104+
105+
echo "[smoke] Reports:"
106+
echo " - JUnit XML : ${XML}"
107+
echo " - HTML : ${HTML}"
108+
echo " - Log : ${LOG}"

0 commit comments

Comments
 (0)