fix(hetzner-e2e): classify 403 limit_reached as quota_exceeded, not missing_token

Run 26077765355 surfaced a misleading "missing_token" error on a Hetzner
project that actually hit its server cap. The API returned HTTP 403 with
body `{ error: { code: "limit_reached", message: "server limit reached" } }`,
but `mapStatusToCode` short-circuited on status 401/403 before reading the
apiCode, collapsing quota exhaustion and missing-token into one bucket.

Layer 1: reorder `mapStatusToCode` so the explicit `limit_reached` /
`resource_limit_exceeded` apiCode wins over the status-only auth fallback.
This is the only correct mapping: status 403 with `limit_reached` is a
project quota issue, not an auth problem, and operators should not be
told to refresh a token that is working fine.

Provision script: extend `isRetryableCombo` to also retry the next
fallback combo on `quota_exceeded` (and the literal "server limit
reached" / "limit_reached" / "resource_limit_exceeded" message strings,
so we stay correct if the error is rewrapped). The fallback ladder is
finite, so a genuinely exhausted project still surfaces a single, clear
error after the ladder is exhausted — with an operator-facing hint
("delete leaked CI servers in https://console.hetzner.cloud/ or rotate
HCLOUD_TOKEN_CI to a project with capacity"). A non-retryable 401/403
also now prints a hint pointing at the GitHub `ci-hetzner-e2e`
environment.

Layer 2: workflow now captures the provision step's combined output to
a log and, on failure, writes a categorized diagnostic to
GITHUB_STEP_SUMMARY (quota vs auth vs unknown) so the operator opening
the failed run sees the actionable next step at the top instead of
buried in the step log.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: lalalune

.github/workflows/hetzner-e2e.yml

@@ -96,8 +96,39 @@ jobs:
         run: bun install --no-save --ignore-scripts
 
       - name: Provision Hetzner server
+        id: provision
         if: steps.secret_config.outputs.configured == 'true'
-        run: bun run packages/scripts/cloud/admin/hetzner-e2e/hetzner-e2e-provision.ts
+        run: |
+          set -o pipefail
+          log=/tmp/hetzner-e2e-provision.log
+          bun run packages/scripts/cloud/admin/hetzner-e2e/hetzner-e2e-provision.ts 2>&1 | tee "$log"
+
+      - name: Surface provision failure diagnostic
+        if: always() && steps.secret_config.outputs.configured == 'true' && steps.provision.outcome == 'failure'
+        run: |
+          log=/tmp/hetzner-e2e-provision.log
+          {
+            echo "### Hetzner E2E provisioning failed"
+            echo ""
+            if [ -f "$log" ] && grep -qE "quota exhausted|server limit reached|limit_reached|resource_limit_exceeded|quota_exceeded" "$log"; then
+              echo "**Cause:** Hetzner project quota exhausted (server cap reached)."
+              echo ""
+              echo "Operator action:"
+              echo "1. Open https://console.hetzner.cloud/ and check the CI project for leaked servers (filter labels \`ci=true\`, \`workflow=hetzner-e2e\`)."
+              echo "2. Delete any leaked servers, or wait for the half-hourly reaper workflow to sweep them."
+              echo "3. Re-run this workflow."
+              echo ""
+              echo "If the project itself has a tighter cap than the fallback ladder can survive, request a quota increase from Hetzner support or rotate \`HCLOUD_TOKEN_CI\` to a project with capacity."
+            elif [ -f "$log" ] && grep -qE "Hetzner rejected the token|missing_token|HTTP 401|HTTP 403" "$log"; then
+              echo "**Cause:** Hetzner rejected the API token (HTTP 401/403)."
+              echo ""
+              echo "Operator action: refresh \`HCLOUD_TOKEN_CI\` in the \`ci-hetzner-e2e\` GitHub environment, or confirm the project is still active."
+            else
+              echo "**Cause:** see the \"Provision Hetzner server\" step log for the raw error."
+              echo ""
+              echo "If the error message is unfamiliar, check \`packages/scripts/cloud/admin/hetzner-e2e/README.md\` and the fallback ladder in \`hetzner-e2e-provision.ts\`."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Wait for host ready
         if: steps.secret_config.outputs.configured == 'true'

packages/cloud-shared/src/lib/services/containers/hetzner-cloud-api.ts

@@ -460,13 +460,19 @@ export class HetznerCloudClient {
 // ---------------------------------------------------------------------------
 
 function mapStatusToCode(status: number, apiCode?: string): HetznerCloudErrorCode {
+  // Explicit quota/limit apiCodes win over auth-status fallback: Hetzner
+  // returns HTTP 403 with body code `limit_reached` (or
+  // `resource_limit_exceeded`) when the project's server cap is hit. Without
+  // this priority, `status === 403` collapses both "no token" and "quota
+  // exhausted" into `missing_token`, which sends operators chasing a
+  // non-existent auth bug while the real issue is account quota.
+  if (apiCode === "limit_reached" || apiCode === "resource_limit_exceeded") {
+    return "quota_exceeded";
+  }
   if (status === 404) return "not_found";
   if (status === 401 || status === 403) return "missing_token";
   if (status === 422 || status === 400) return "invalid_input";
   if (status === 429) return "rate_limited";
-  if (apiCode === "limit_reached" || apiCode === "resource_limit_exceeded") {
-    return "quota_exceeded";
-  }
   return "server_error";
 }
 

packages/scripts/cloud/admin/hetzner-e2e/hetzner-e2e-provision.ts

@@ -15,7 +15,10 @@
  * any further work — so a crash never leaks a server.
  */
 
-import { HetznerCloudClient } from "@elizaos/cloud-shared/lib/services/containers/hetzner-cloud-api";
+import {
+  HetznerCloudClient,
+  HetznerCloudError,
+} from "@elizaos/cloud-shared/lib/services/containers/hetzner-cloud-api";
 import { appendStateAtomic } from "./state-file";
 
 function requireEnv(name: string): string {
@@ -48,10 +51,18 @@ const SERVER_TYPE_FALLBACKS: ReadonlyArray<{
 // Hetzner's "this server type can't be created here" and "this server
 // type is going away" responses — both render the requested combo
 // unusable and a different shared-cpu type / location is the natural
-// remediation. Auth / quota / billing failures (HTTP 401/402/403) are
-// NOT in this list — those are surfaced unchanged so the operator
+// remediation. We also treat project-wide quota exhaustion as retryable
+// because the pre-reap pass runs immediately before the loop: if it
+// freed any slots, the next attempt may now fit under the cap. The
+// fallback ladder is finite (~5 combos) so a genuinely exhausted project
+// will still surface as the last combo's error after the loop exits.
+// Pure auth / billing failures (HTTP 401, real 403 without limit code)
+// are NOT in this list — those are surfaced unchanged so the operator
 // fixes the underlying account issue.
 function isRetryableCombo(err: unknown): boolean {
+  if (err instanceof HetznerCloudError && err.code === "quota_exceeded") {
+    return true;
+  }
   const message = err instanceof Error ? err.message.toLowerCase() : "";
   return (
     message.includes("unsupported_server_type_for_location") ||
@@ -60,7 +71,14 @@ function isRetryableCombo(err: unknown): boolean {
     message.includes("is deprecated") ||
     message.includes("server_type_deprecated") ||
     message.includes("resource_unavailable") ||
-    message.includes("not_found") // Hetzner returns 404 when a deprecated type is fully removed
+    message.includes("not_found") || // Hetzner returns 404 when a deprecated type is fully removed
+    // Hetzner returns HTTP 403 with body `{ error: { code: "limit_reached",
+    // message: "server limit reached" } }` when the project cap is hit.
+    // Match both the apiCode and the human message so we stay correct even
+    // if mapStatusToCode is bypassed (e.g. transport layer wraps the body).
+    message.includes("server limit reached") ||
+    message.includes("limit_reached") ||
+    message.includes("resource_limit_exceeded")
   );
 }
 
@@ -179,14 +197,47 @@ async function main(): Promise<void> {
       break;
     } catch (err) {
       lastError = err;
-      if (!isRetryableCombo(err)) throw err;
+      if (!isRetryableCombo(err)) {
+        // Surface a hint before propagating: a non-retryable failure on
+        // the first attempt is almost always an auth/account problem
+        // (missing or stale HCLOUD_TOKEN_CI, project disabled). Without
+        // this, the workflow log just shows the bare HetznerCloudError
+        // and the operator has to guess.
+        if (
+          err instanceof HetznerCloudError &&
+          err.code === "missing_token"
+        ) {
+          console.error(
+            "[hetzner-e2e-provision] Hetzner rejected the token (HTTP 401/403). " +
+              "Refresh HCLOUD_TOKEN_CI in the ci-hetzner-e2e GitHub environment, or verify the project is active.",
+          );
+        }
+        throw err;
+      }
       const reason = err instanceof Error ? err.message : String(err);
       console.error(
         `[hetzner-e2e-provision] ${attempt.serverType}@${attempt.location} unavailable (${reason}); trying next fallback`,
       );
     }
   }
   if (!server) {
+    // Layer 2 diagnostic: when every fallback combo also failed with
+    // quota_exceeded, the operator needs a single actionable next step —
+    // not a stack of "unavailable" lines that look like a transient API
+    // glitch. Refresh `HCLOUD_TOKEN_CI` only if the token's project no
+    // longer matches the CI project; otherwise delete leaked servers in
+    // the Hetzner console (https://console.hetzner.cloud/) and re-run.
+    if (
+      lastError instanceof HetznerCloudError &&
+      lastError.code === "quota_exceeded"
+    ) {
+      throw new Error(
+        `Hetzner project quota exhausted across all fallback combos (last error: ${lastError.message}). ` +
+          "Operator action required: pre-reap freed nothing in 20min window, and the workflow cannot proceed. " +
+          "Check https://console.hetzner.cloud/ for leaked CI servers (label ci=true, workflow=hetzner-e2e), " +
+          "or rotate HCLOUD_TOKEN_CI if it now points to a project with a tighter cap.",
+      );
+    }
     throw lastError instanceof Error
       ? lastError
       : new Error("Hetzner provisioning failed across all fallback combos");