Skip to content

Commit d35cbe4

Browse files
committed
test(e2e): drop alpine literals, skip local-only verifier, modernize id check, expand purge to cpu=999
Five wins toward a green e2e suite on Tokyo: 1. test_quota_enforcement.py + test_error_code_mapping.py: every hardcoded 'alpine:3.23' literal in a POST body switched to conftest.DEFAULT_IMAGE. The alpine string was rejected by the API image allowlist (HTTP 400 'Unsupported image') before reaching the actual cpu/memory/disk quota check, so each xfail-strict case passed-by-accident and tripped XPASS(strict). Sending the curated ghcr digest lets the request reach the quota path the test is meant to pin — which still silently clamps, so the cases now fail as expected and xfail captures them. 2. test_path_verification.py: module-level skipif when BOXLITE_E2E_PROFILE != 'default'. Both cases are LOCAL meta-tests (one greps credentials.toml for url=:3000, the other reads the boxlite-runner systemd journal) and can never succeed on the Tokyo cloud profile. 3. test_lifecycle::test_create_generates_unique_ids: post-#735 box ids are 12-char alphanumeric (BOX_ID_REGEX), no longer the 5-segment uuid the assertion was checking. 4. ci/e2e-cloud-test.yml purge step extends to rows whose cpu equals the over-quota literal (999) the XPASS tests POSTed back when the API didn't enforce. Those rows are still in the box table and blow up every list_info round-trip because the Python SDK types BoxInfo.cpus as u8 ('invalid value: integer 999, expected u8'). The 999 literal is provably from test pollution — way past max_cpu_per_box=4 and the ADMIN_TOTAL_CPU_QUOTA=32 envelope. 5. .gitignore: add .claude/.pr-reviewed.json so the claude review-state file isn't accidentally staged into review commits.
1 parent 53c53ff commit d35cbe4

6 files changed

Lines changed: 67 additions & 25 deletions

File tree

.github/workflows/e2e-cloud-test.yml

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -246,17 +246,28 @@ jobs:
246246
| grep -E '\|4\|8192\|20$' \
247247
|| { echo "::error::admin-org quota SELECT did not show target values (|4|8192|20)"; exit 1; }
248248
249-
- name: Purge stale named-fixture box rows (idempotent)
250-
# test_create_named_box uses a stable name ('e2e-test-box') and
251-
# cleans up via `auto_remove=True` only when the test reaches
252-
# its `finally`. Any prior run that failed before that point
253-
# (e.g. runs from before the image-pull fix in libboxlite/store)
254-
# leaves the row behind, and the next run then 409s with
255-
# "Box with name e2e-test-box already exists" at [1%].
256-
# Surgical delete: only the named-fixture rows on the admin's
257-
# default org (no broad wipe — preserves any in-flight test
258-
# state for unrelated suites). box_last_activity / ssh_access
259-
# cascade-delete via FK.
249+
- name: Purge stale e2e box rows (idempotent)
250+
# Two distinct cleanup problems on the shared Tokyo CI DB:
251+
#
252+
# (1) test_create_named_box uses the stable name 'e2e-test-box'
253+
# and only cleans up in its `finally`. A prior run that died
254+
# before that point leaves the row behind, and the next run
255+
# 409s at [1%].
256+
#
257+
# (2) Earlier runs of the XPASS-strict quota tests POSTed
258+
# `cpus=999` against the API back when there was no boundary
259+
# enforcement, leaving Box rows whose `cpu` column is 999.
260+
# The Python SDK's BoxInfo struct types `cpus: u8` (max 255),
261+
# so every test_list_info* that round-trips the response
262+
# blows up with `invalid value: integer 999, expected u8`.
263+
#
264+
# Surgical, provably-test-only scope on the admin's default org:
265+
# - the fixture name
266+
# - the exact literal cpu value the over-quota cases POST
267+
# (999 — way outside max_cpu_per_box=4 and the
268+
# ADMIN_TOTAL_CPU_QUOTA=32 envelope, so it cannot be
269+
# legitimate state).
270+
# box_last_activity / ssh_access cascade-delete via FK.
260271
run: |
261272
set -euo pipefail
262273
CLUSTER="${{ steps.resources.outputs.cluster }}"
@@ -271,15 +282,15 @@ jobs:
271282
if [ "$TD" = "$PRIMARY_TD" ]; then echo "$arn"; break; fi
272283
done)
273284
[ -n "$TASK" ] || { echo "::error::No Api task on PRIMARY deployment"; exit 1; }
274-
SQL='DELETE FROM "box" WHERE "name" = '\''e2e-test-box'\'' AND "organizationId" = (SELECT "organizationId" FROM "organization_user" WHERE "userId" = '\''boxlite-admin'\'' AND "isDefaultForUser" = true LIMIT 1); SELECT count(*) FROM "box" WHERE "name" = '\''e2e-test-box'\'' AND "organizationId" = (SELECT "organizationId" FROM "organization_user" WHERE "userId" = '\''boxlite-admin'\'' AND "isDefaultForUser" = true LIMIT 1);'
285+
SQL='DELETE FROM "box" WHERE "organizationId" = (SELECT "organizationId" FROM "organization_user" WHERE "userId" = '\''boxlite-admin'\'' AND "isDefaultForUser" = true LIMIT 1) AND ("name" = '\''e2e-test-box'\'' OR "cpu" = 999); SELECT count(*) FROM "box" WHERE "organizationId" = (SELECT "organizationId" FROM "organization_user" WHERE "userId" = '\''boxlite-admin'\'' AND "isDefaultForUser" = true LIMIT 1) AND ("name" = '\''e2e-test-box'\'' OR "cpu" = 999);'
275286
SQL_B64=$(printf '%s' "$SQL" | base64 -w0)
276287
OUT=/tmp/ecs_purge.log
277288
aws ecs execute-command --cluster "$CLUSTER" --task "$TASK" --container Api --interactive \
278289
--command "sh -c \"echo ${SQL_B64} | base64 -d | PAGER=cat PGPASSWORD=\\\"\\\$DB_PASSWORD\\\" psql -h \\\"\\\$DB_HOST\\\" -U \\\"\\\$DB_USERNAME\\\" -d \\\"\\\$DB_DATABASE\\\" -A -t -P pager=off -v ON_ERROR_STOP=1 -f -\"" \
279290
2>&1 | tee "$OUT"
280-
# Post-purge: zero rows with the fixture name on this org.
291+
# Post-purge: zero rows matching the wipe predicate on this org.
281292
tr -d '\r' < "$OUT" | grep -E '^0$' \
282-
|| { echo "::error::post-purge SELECT count for e2e-test-box did not show 0"; exit 1; }
293+
|| { echo "::error::post-purge SELECT count did not show 0"; exit 1; }
283294
284295
# ──────────────────────────────────────────────────────────────────
285296
# Build SDK from THIS checkout (path_prefix and other source-tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ Cargo.lock
179179
.claude/scheduled_tasks.lock
180180
.claude/worktrees/
181181
.claude/.last-audit.json
182+
.claude/.pr-reviewed.json
182183

183184
# ============================================================================
184185
# Local dev artifacts (apps/infra-local + dashboard E2E session)

scripts/test/e2e/cases/test_error_code_mapping.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import boxlite
3434
import pytest
3535

36+
from conftest import DEFAULT_IMAGE
37+
3638

3739
def _profile() -> dict:
3840
return tomllib.loads((Path.home() / ".boxlite/credentials.toml").read_text())[
@@ -113,7 +115,7 @@ async def test_invalid_argument_zero_cpu_returns_400(rt):
113115
status, body = _api_call(
114116
"POST",
115117
f"/v1/{p['path_prefix']}/boxes",
116-
{"image": "alpine:3.23", "cpus": 0, "memory_mib": 256, "disk_size_gb": 4},
118+
{"image": DEFAULT_IMAGE, "cpus": 0, "memory_mib": 256, "disk_size_gb": 4},
117119
)
118120
_assert_http_code(
119121
status,
@@ -141,7 +143,7 @@ async def test_invalid_argument_negative_memory_returns_400(rt):
141143
status, body = _api_call(
142144
"POST",
143145
f"/v1/{p['path_prefix']}/boxes",
144-
{"image": "alpine:3.23", "cpus": 1, "memory_mib": -1, "disk_size_gb": 4},
146+
{"image": DEFAULT_IMAGE, "cpus": 1, "memory_mib": -1, "disk_size_gb": 4},
145147
)
146148
_assert_http_code(
147149
status,
@@ -254,7 +256,7 @@ async def test_resource_exhausted_over_cpu_quota_returns_429(rt):
254256
status, body = _api_call(
255257
"POST",
256258
f"/v1/{p['path_prefix']}/boxes",
257-
{"image": "alpine:3.23", "cpus": 999, "memory_mib": 256, "disk_size_gb": 4},
259+
{"image": DEFAULT_IMAGE, "cpus": 999, "memory_mib": 256, "disk_size_gb": 4},
258260
)
259261
# The mapping says 429 ResourceExhausted; some implementations may also
260262
# 400 InvalidArgument (treating it as a parse-time validation failure).

scripts/test/e2e/cases/test_lifecycle.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,15 @@ async def test_create_generates_unique_ids(rt, image):
3030
b = await rt.create(boxlite.BoxOptions(image=image, auto_remove=True))
3131
try:
3232
assert a.id != b.id
33-
# uuid v4 format check
34-
assert len(a.id.split("-")) == 5
35-
assert len(b.id.split("-")) == 5
33+
# Post-#735 box ids are 12-char alphanumeric (BOX_ID_REGEX in
34+
# apps/api/src/box/utils/box-id.util.ts) — no longer the
35+
# 5-segment uuid format the pre-collapse SDK was returning.
36+
import re
37+
for box_id in (a.id, b.id):
38+
assert re.fullmatch(r"[0-9A-Za-z]{12}", box_id), (
39+
f"box id {box_id!r} doesn't match the post-#735 "
40+
f"12-char alphanumeric BOX_ID_REGEX"
41+
)
3642
finally:
3743
await rt.remove(a.id, force=True)
3844
await rt.remove(b.id, force=True)

scripts/test/e2e/cases/test_path_verification.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"""
1919
from __future__ import annotations
2020

21+
import os
2122
import sys
2223
from pathlib import Path
2324

@@ -28,6 +29,25 @@
2829
from path_verification import runner_journal_seek, runner_hits_for_box
2930
from conftest import drain
3031

32+
# Both cases in this file are LOCAL-only meta-tests:
33+
# (1) inspects ~/.boxlite/credentials for url=':3000' — only true for
34+
# the local-bootstrap profile, never for a cloud profile pointing at
35+
# the ELB DNS.
36+
# (2) reads the local boxlite-runner systemd journal via journalctl —
37+
# on the Tokyo cloud profile p1 the runner journal lives on an
38+
# EC2 instance the test client can't reach.
39+
# Skip the whole module when the test session is configured against
40+
# anything other than the default (= local) profile so these don't
41+
# spam the cloud failure tally with environment-mismatch noise.
42+
pytestmark = pytest.mark.skipif(
43+
os.environ.get("BOXLITE_E2E_PROFILE", "default") != "default",
44+
reason=(
45+
"LOCAL-only meta-test (checks credentials.toml url=:3000 and "
46+
"reads host's boxlite-runner journalctl). Cannot run against a "
47+
"remote profile."
48+
),
49+
)
50+
3151

3252
@pytest.mark.asyncio
3353
async def test_sdk_runtime_is_rest_against_local_api(rt):

scripts/test/e2e/cases/test_quota_enforcement.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737

3838
import pytest
3939

40+
from conftest import DEFAULT_IMAGE
41+
4042
pytestmark = pytest.mark.xfail(
4143
strict=True,
4244
reason=(
@@ -94,7 +96,7 @@ def _delete_box(box_id: str) -> None:
9496
async def test_cpus_above_per_box_limit_returns_4xx():
9597
"""cpus far above max_cpu_per_box (4) → 429 or 400, not 5xx."""
9698
status, body = _post_box(
97-
{"image": "alpine:3.23", "cpus": 999, "memory_mib": 256, "disk_size_gb": 4}
99+
{"image": DEFAULT_IMAGE, "cpus": 999, "memory_mib": 256, "disk_size_gb": 4}
98100
)
99101
body_str = json.dumps(body) if body else ""
100102
assert 400 <= status < 500, f"cpus=999 leaked HTTP {status}: {body_str}"
@@ -105,7 +107,7 @@ async def test_memory_above_per_box_limit_returns_4xx():
105107
"""memory far above max_memory_per_box (8 GiB) → 4xx, not 5xx."""
106108
status, body = _post_box(
107109
{
108-
"image": "alpine:3.23",
110+
"image": DEFAULT_IMAGE,
109111
"cpus": 1,
110112
"memory_mib": 8_192_000_000,
111113
"disk_size_gb": 4,
@@ -120,7 +122,7 @@ async def test_disk_above_per_box_limit_returns_4xx():
120122
"""disk far above max_disk_per_box (20 GiB) → 4xx, not 5xx."""
121123
status, body = _post_box(
122124
{
123-
"image": "alpine:3.23",
125+
"image": DEFAULT_IMAGE,
124126
"cpus": 1,
125127
"memory_mib": 256,
126128
"disk_size_gb": 99_999_999,
@@ -136,7 +138,7 @@ async def test_quota_violation_does_not_silently_create_box(rt):
136138
immediately and find an orphan with cpus=999, the runner accepted the
137139
doomed request and the quota check is decorative."""
138140
status, body = _post_box(
139-
{"image": "alpine:3.23", "cpus": 999, "memory_mib": 256, "disk_size_gb": 4}
141+
{"image": DEFAULT_IMAGE, "cpus": 999, "memory_mib": 256, "disk_size_gb": 4}
140142
)
141143
if 200 <= status < 300:
142144
pytest.fail(f"cpus=999 unexpectedly succeeded: HTTP {status}, body={body}")
@@ -158,7 +160,7 @@ async def test_quota_zero_cpus_returns_4xx():
158160
"""cpus=0 — boundary at the other end. Must be 4xx, not 500 or a box
159161
that immediately crashes."""
160162
status, body = _post_box(
161-
{"image": "alpine:3.23", "cpus": 0, "memory_mib": 256, "disk_size_gb": 4}
163+
{"image": DEFAULT_IMAGE, "cpus": 0, "memory_mib": 256, "disk_size_gb": 4}
162164
)
163165
body_str = json.dumps(body) if body else ""
164166
assert 400 <= status < 500, f"cpus=0 leaked HTTP {status}: {body_str}"

0 commit comments

Comments
 (0)