Skip to content

Commit 31df427

Browse files
drdrew42claude
drdrew42
and
claude
committed
Add CI integration test suite
Bash + curl + jq integration tests that run against the Docker container. Four test suites validate the render API end-to-end: - Smoke: health check, response structure, instructor vs student fields - Render parity: raw params vs JWE/JWS JWT produce identical content - Answer cycle: correct/wrong scoring, partial credit, preview mode - Endpoints: IO routes, error handling, malformed JWT, simultaneous submit+preview rejection PG unit tests run for visibility but do not gate CI (upstream's responsibility). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 3451dd7 commit 31df427

9 files changed

Lines changed: 843 additions & 0 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Integration Tests
2+
3+
on:
4+
push:
5+
branches: [main, development]
6+
pull_request:
7+
branches: [main, development]
8+
9+
jobs:
10+
test:
11+
runs-on: ubuntu-latest
12+
timeout-minutes: 20
13+
14+
steps:
15+
- name: Checkout with submodules
16+
uses: actions/checkout@v4
17+
with:
18+
submodules: recursive
19+
20+
- name: Build Docker image
21+
run: docker build -t renderer-test .
22+
23+
- name: Start container
24+
run: |
25+
docker run -d \
26+
--name renderer-test \
27+
-p 3000:3000 \
28+
-e MOJO_MODE=development \
29+
-v "${{ github.workspace }}/t/ci/fixtures:/usr/app/private/test:ro" \
30+
renderer-test \
31+
morbo -l 'http://*:3000' ./script/renderer
32+
33+
- name: Wait for health
34+
run: |
35+
for i in $(seq 1 30); do
36+
if curl -sf --max-time 5 http://localhost:3000/health >/dev/null 2>&1; then
37+
echo "Renderer healthy after $((i * 2))s"
38+
exit 0
39+
fi
40+
sleep 2
41+
done
42+
echo "Renderer failed to start"
43+
docker logs renderer-test 2>&1 | tail -50
44+
exit 1
45+
46+
- name: PG unit tests (informational)
47+
continue-on-error: true
48+
run: |
49+
docker exec renderer-test bash -c \
50+
'export PG_ROOT=/usr/app/lib/PG && cd $PG_ROOT && prove -lr \
51+
t/macros t/contexts t/math_objects t/pg_problems t/units'
52+
53+
- name: Smoke tests
54+
run: bash t/ci/01-smoke.sh
55+
56+
- name: Render parity tests
57+
run: bash t/ci/02-render-parity.sh
58+
59+
- name: Answer cycle tests
60+
run: bash t/ci/03-answer-cycle.sh
61+
62+
- name: Endpoint tests
63+
run: bash t/ci/04-endpoints.sh
64+
65+
- name: Cleanup
66+
if: always()
67+
run: |
68+
docker stop renderer-test 2>/dev/null || true
69+
docker rm renderer-test 2>/dev/null || true

‎t/ci/01-smoke.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env bash
2+
# 01-smoke.sh — Health check, basic render, response structure validation.
3+
4+
set -euo pipefail
5+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
source "$SCRIPT_DIR/lib/helpers.sh"
7+
source "$SCRIPT_DIR/lib/problems.sh"
8+
9+
echo "=== Smoke Tests ==="
10+
11+
# Health endpoint
12+
assert_status "GET" "/health" "200" "GET /health returns 200"
13+
14+
# Editor UI available in dev mode
15+
assert_status "GET" "/" "200" "GET / (editor UI) returns 200 in dev mode"
16+
17+
# Basic render
18+
RESP=$(render_raw "problemSource=${PROBLEM_BASIC}" "problemSeed=42")
19+
20+
assert_json_field "$RESP" '.renderedHTML' "Response has .renderedHTML"
21+
assert_json_field "$RESP" '.JWT.problem' "Response has .JWT.problem (non-empty)"
22+
assert_json_field "$RESP" '.JWT.session' "Response has .JWT.session (non-empty)"
23+
24+
# Score should be 0 (no answers submitted)
25+
SCORE=$(echo "$RESP" | jq -r '.problem_result.score // empty')
26+
assert_eq "$SCORE" "0" "Score is 0 with no answers submitted"
27+
28+
# Debug block should exist
29+
assert_json_field "$RESP" '.debug' "Response has .debug block"
30+
31+
# Resources block should exist
32+
assert_json_field "$RESP" '.resources' "Response has .resources block"
33+
34+
# renderedHTML should contain a form input for the answer
35+
HTML=$(echo "$RESP" | jq -r '.renderedHTML')
36+
assert_contains "$HTML" "input" "renderedHTML contains input element"
37+
38+
# Instructor mode adds answers and inputs to response
39+
RESP_INST=$(render_raw "problemSource=${PROBLEM_BASIC}" "problemSeed=42" "isInstructor=1")
40+
assert_json_field "$RESP_INST" '.answers' "Instructor response has .answers"
41+
assert_json_field "$RESP_INST" '.inputs' "Instructor response has .inputs"
42+
43+
# Non-instructor should NOT have answers
44+
NO_ANSWERS=$(echo "$RESP" | jq -r '.answers // empty')
45+
assert_eq "$NO_ANSWERS" "" "Non-instructor response has no .answers"
46+
47+
summary "smoke tests"

‎t/ci/02-render-parity.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env bash
2+
# 02-render-parity.sh — Verify raw params vs JWT produce identical rendered output.
3+
4+
set -euo pipefail
5+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
source "$SCRIPT_DIR/lib/helpers.sh"
7+
source "$SCRIPT_DIR/lib/problems.sh"
8+
9+
echo "=== Render Parity Tests ==="
10+
11+
# Helper: compare raw render vs JWE render for a given set of params.
12+
# Usage: parity_test DESCRIPTION PARAM...
13+
parity_test_jwe() {
14+
local desc="$1"
15+
shift
16+
local raw_resp jwe_resp raw_hash jwe_hash
17+
18+
raw_resp=$(render_raw "$@")
19+
jwe_resp=$(render_via_jwe "$@")
20+
21+
raw_hash=$(hash_html "$raw_resp")
22+
jwe_hash=$(hash_html "$jwe_resp")
23+
24+
assert_eq "$jwe_hash" "$raw_hash" "JWE parity: $desc"
25+
}
26+
27+
parity_test_jws() {
28+
local desc="$1"
29+
shift
30+
local raw_resp jws_resp raw_hash jws_hash
31+
32+
raw_resp=$(render_raw "$@")
33+
jws_resp=$(render_via_jws "$@")
34+
35+
raw_hash=$(hash_html "$raw_resp")
36+
jws_hash=$(hash_html "$jws_resp")
37+
38+
assert_eq "$jws_hash" "$raw_hash" "JWS parity: $desc"
39+
}
40+
41+
# Basic problem, default params
42+
parity_test_jwe "basic problem, defaults" \
43+
"problemSource=${PROBLEM_BASIC}" "problemSeed=42"
44+
45+
# Basic problem with isInstructor
46+
parity_test_jwe "basic problem, isInstructor=1" \
47+
"problemSource=${PROBLEM_BASIC}" "problemSeed=42" "isInstructor=1"
48+
49+
# Basic problem with showCorrectAnswers
50+
parity_test_jwe "basic problem, instructor + showCorrectAnswers" \
51+
"problemSource=${PROBLEM_BASIC}" "problemSeed=42" "isInstructor=1" "showCorrectAnswers=1"
52+
53+
# Seed-sensitive problem with seed=42
54+
parity_test_jwe "random problem, seed=42" \
55+
"problemSource=${PROBLEM_RANDOM}" "problemSeed=42"
56+
57+
# Seed-sensitive problem with seed=99999
58+
parity_test_jwe "random problem, seed=99999" \
59+
"problemSource=${PROBLEM_RANDOM}" "problemSeed=99999"
60+
61+
# Multi-answer problem
62+
parity_test_jwe "multi-answer problem" \
63+
"problemSource=${PROBLEM_MULTI}" "problemSeed=42"
64+
65+
# JWS parity (HS256 signed instead of encrypted)
66+
parity_test_jws "basic problem via JWS" \
67+
"problemSource=${PROBLEM_BASIC}" "problemSeed=42"
68+
69+
# Verify different seeds produce different output
70+
RESP_42=$(render_raw "problemSource=${PROBLEM_RANDOM}" "problemSeed=42")
71+
RESP_99=$(render_raw "problemSource=${PROBLEM_RANDOM}" "problemSeed=99999")
72+
HASH_42=$(hash_html "$RESP_42")
73+
HASH_99=$(hash_html "$RESP_99")
74+
assert_ne "$HASH_42" "$HASH_99" "Different seeds produce different renderedHTML"
75+
76+
summary "render parity tests"

‎t/ci/03-answer-cycle.sh‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/usr/bin/env bash
2+
# 03-answer-cycle.sh — Render → submit → verify scoring.
3+
4+
set -euo pipefail
5+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
6+
source "$SCRIPT_DIR/lib/helpers.sh"
7+
source "$SCRIPT_DIR/lib/problems.sh"
8+
9+
echo "=== Answer Cycle Tests ==="
10+
11+
# ── Single answer: correct ────────────────────────────────────
12+
13+
# Step 1: Render as instructor to discover answer field name + correct value
14+
INST_RESP=$(render_raw "problemSource=${PROBLEM_BASIC}" "problemSeed=42" "isInstructor=1")
15+
16+
# Extract the answer field name (e.g., AnSwEr0001)
17+
ANS_NAME=$(echo "$INST_RESP" | jq -r '.answers | keys[0]')
18+
CORRECT_VAL=$(echo "$INST_RESP" | jq -r ".answers.\"$ANS_NAME\".correct_ans")
19+
20+
assert_ne "$ANS_NAME" "" "Discovered answer field name: $ANS_NAME"
21+
assert_ne "$CORRECT_VAL" "" "Discovered correct answer: $CORRECT_VAL"
22+
23+
# Step 2: Render as student to get JWT tokens
24+
STU_RESP=$(render_raw "problemSource=${PROBLEM_BASIC}" "problemSeed=42")
25+
PROB_JWT=$(echo "$STU_RESP" | jq -r '.JWT.problem')
26+
SESS_JWT=$(echo "$STU_RESP" | jq -r '.JWT.session')
27+
28+
# Step 3: Submit correct answer
29+
SUBMIT_RESP=$(curl -sf --max-time "$CURL_TIMEOUT" -X POST \
30+
-d "problemJWT=${PROB_JWT}" \
31+
-d "sessionJWT=${SESS_JWT}" \
32+
-d "${ANS_NAME}=${CORRECT_VAL}" \
33+
-d "submitAnswers=1" \
34+
-d "answersSubmitted=1" \
35+
-d "_format=json" \
36+
"${BASE_URL}/render-api" 2>/dev/null)
37+
38+
SCORE=$(echo "$SUBMIT_RESP" | jq -r '.problem_result.score')
39+
assert_eq "$SCORE" "1" "Correct answer scores 1"
40+
41+
# Step 4: Submit wrong answer
42+
WRONG_RESP=$(curl -sf --max-time "$CURL_TIMEOUT" -X POST \
43+
-d "problemJWT=${PROB_JWT}" \
44+
-d "sessionJWT=${SESS_JWT}" \
45+
-d "${ANS_NAME}=999" \
46+
-d "submitAnswers=1" \
47+
-d "answersSubmitted=1" \
48+
-d "_format=json" \
49+
"${BASE_URL}/render-api" 2>/dev/null)
50+
51+
WRONG_SCORE=$(echo "$WRONG_RESP" | jq -r '.problem_result.score')
52+
assert_eq "$WRONG_SCORE" "0" "Wrong answer scores 0"
53+
54+
# ── Multi-answer: partial credit ──────────────────────────────
55+
56+
MULTI_INST=$(render_raw "problemSource=${PROBLEM_MULTI}" "problemSeed=42" "isInstructor=1")
57+
58+
# Get both answer field names and correct values
59+
ANS1_NAME=$(echo "$MULTI_INST" | jq -r '.answers | keys[0]')
60+
ANS2_NAME=$(echo "$MULTI_INST" | jq -r '.answers | keys[1]')
61+
ANS1_CORRECT=$(echo "$MULTI_INST" | jq -r ".answers.\"$ANS1_NAME\".correct_ans")
62+
ANS2_CORRECT=$(echo "$MULTI_INST" | jq -r ".answers.\"$ANS2_NAME\".correct_ans")
63+
64+
# Render as student
65+
MULTI_STU=$(render_raw "problemSource=${PROBLEM_MULTI}" "problemSeed=42")
66+
MULTI_PROB_JWT=$(echo "$MULTI_STU" | jq -r '.JWT.problem')
67+
MULTI_SESS_JWT=$(echo "$MULTI_STU" | jq -r '.JWT.session')
68+
69+
# Submit 1/2 correct (first correct, second wrong)
70+
PARTIAL_RESP=$(curl -sf --max-time "$CURL_TIMEOUT" -X POST \
71+
-d "problemJWT=${MULTI_PROB_JWT}" \
72+
-d "sessionJWT=${MULTI_SESS_JWT}" \
73+
-d "${ANS1_NAME}=${ANS1_CORRECT}" \
74+
-d "${ANS2_NAME}=999" \
75+
-d "submitAnswers=1" \
76+
-d "answersSubmitted=1" \
77+
-d "_format=json" \
78+
"${BASE_URL}/render-api" 2>/dev/null)
79+
80+
PARTIAL_SCORE=$(echo "$PARTIAL_RESP" | jq -r '.problem_result.score')
81+
assert_eq "$PARTIAL_SCORE" "0.5" "1/2 correct scores 0.5"
82+
83+
# Submit both correct
84+
FULL_RESP=$(curl -sf --max-time "$CURL_TIMEOUT" -X POST \
85+
-d "problemJWT=${MULTI_PROB_JWT}" \
86+
-d "sessionJWT=${MULTI_SESS_JWT}" \
87+
-d "${ANS1_NAME}=${ANS1_CORRECT}" \
88+
-d "${ANS2_NAME}=${ANS2_CORRECT}" \
89+
-d "submitAnswers=1" \
90+
-d "answersSubmitted=1" \
91+
-d "_format=json" \
92+
"${BASE_URL}/render-api" 2>/dev/null)
93+
94+
FULL_SCORE=$(echo "$FULL_RESP" | jq -r '.problem_result.score')
95+
assert_eq "$FULL_SCORE" "1" "2/2 correct scores 1"
96+
97+
# ── Preview mode ──────────────────────────────────────────────
98+
99+
# Preview mode renders the formatted input. The grader still runs (score reflects
100+
# correctness), but the rendered HTML shows formatted input rather than right/wrong.
101+
PREVIEW_RESP=$(curl -sf --max-time "$CURL_TIMEOUT" -X POST \
102+
-d "problemJWT=${PROB_JWT}" \
103+
-d "sessionJWT=${SESS_JWT}" \
104+
-d "${ANS_NAME}=${CORRECT_VAL}" \
105+
-d "previewAnswers=1" \
106+
-d "answersSubmitted=1" \
107+
-d "_format=json" \
108+
"${BASE_URL}/render-api" 2>/dev/null)
109+
110+
assert_json_field "$PREVIEW_RESP" '.renderedHTML' "Preview mode returns rendered HTML"
111+
assert_json_field "$PREVIEW_RESP" '.JWT.session' "Preview mode returns session JWT"
112+
113+
summary "answer cycle tests"

0 commit comments

Comments
 (0)