fix(deploy): accept idempotent 200 in tenant bootstrap (catch-all already exists) (#343)

The proxy self-provisions the catch-all tenant at startup from
LLMTRACE_DEFAULT_TENANT_ID (ensure_tenant_exists), so the lifecycle's
subsequent POST /api/v1/tenants for the catch-all returns 200 (already exists),
not 201. _bootstrap_tenant_in_proxy raised on anything != 201, aborting
provisioning BEFORE the dashboard was recreated — a live outage on 2026-05-30.
Accept 200 and 201 (both carry the same body). Regression tests added.

Author: epappas

deployments/basilica/lifecycle.py

@@ -921,7 +921,11 @@ def _bootstrap_tenant_in_proxy(
         admin_key,
         body=body,
     )
-    if status != 201:
+    # Idempotent create: the proxy returns 201 for a newly-created tenant and
+    # 200 when one with this `id` already exists (e.g. the proxy self-provisions
+    # the catch-all tenant at startup from LLMTRACE_DEFAULT_TENANT_ID before the
+    # lifecycle's create runs). Both carry the same body shape; both are success.
+    if status not in (200, 201):
         raise RuntimeError(
             f"tenant bootstrap failed: status={status} body={payload}"
         )

deployments/basilica/tests/test_catchall_and_operator_tenant.py

@@ -367,3 +367,44 @@ def test_pro_yaml_has_no_hardcoded_tenant_uuid() -> None:
         "the previously-hardcoded catch-all/default tenant UUID must not "
         "appear anywhere in pro.yaml"
     )
+
+
+def _patch_admin_http(monkeypatch: Any, status: int) -> None:
+    def fake(
+        proxy_url: str,
+        path: str,
+        method: str,
+        admin_key: str,
+        body: Optional[dict[str, Any]] = None,
+    ) -> tuple[int, dict[str, Any]]:
+        body = body or {}
+        return status, {"id": body.get("id") or "generated-id", "name": body.get("name")}
+
+    monkeypatch.setattr(lifecycle, "_admin_http_request", fake)
+
+
+def test_bootstrap_tenant_accepts_idempotent_200(monkeypatch: Any) -> None:
+    """Regression for the 2026-05-30 outage: the proxy self-provisions the
+    catch-all at startup from LLMTRACE_DEFAULT_TENANT_ID, so the lifecycle's
+    subsequent create gets 200 (already exists). 200 MUST be treated as success
+    (the previous code raised on anything != 201, aborting before the dashboard
+    was recreated)."""
+    _patch_admin_http(monkeypatch, 200)
+    tid = lifecycle._bootstrap_tenant_in_proxy(
+        "http://proxy", "admin", "catch-all", stable_id="a0f7d8a5-4524-4a83-afac-27080b4d0432"
+    )
+    assert tid == "a0f7d8a5-4524-4a83-afac-27080b4d0432"
+
+
+def test_bootstrap_tenant_accepts_created_201(monkeypatch: Any) -> None:
+    _patch_admin_http(monkeypatch, 201)
+    tid = lifecycle._bootstrap_tenant_in_proxy(
+        "http://proxy", "admin", "operator", stable_id="550e8400-e29b-41d4-a716-446655440000"
+    )
+    assert tid == "550e8400-e29b-41d4-a716-446655440000"
+
+
+def test_bootstrap_tenant_rejects_error_status(monkeypatch: Any) -> None:
+    _patch_admin_http(monkeypatch, 500)
+    with pytest.raises(RuntimeError, match="tenant bootstrap failed"):
+        lifecycle._bootstrap_tenant_in_proxy("http://proxy", "admin", "catch-all")