fix(daemon): retry deps init with backoff to recover from transient failures

daemon/main.ts

@@ -271,29 +271,30 @@ const createDeps = (
 ): MiddlewareHandler & { ready: Promise<void>; ensureStarted: () => void } => {
   let ok: Promise<unknown> | null = null;
   let readyResolve: () => void;
-  let readyReject: (err: unknown) => void;
-  const ready = new Promise<void>((resolve, reject) => {
+  // `ready` is intentionally never rejected. We resolve it on the first
+  // successful `ensureGit` (inside `runInitSteps`); failures are surfaced
+  // to the caller via the `ok` Promise instead. Keeping `ready` settle-once
+  // and resolve-only means consumers that await it (e.g. worker factories)
+  // can recover after a cooldown-driven retry.
+  const ready = new Promise<void>((resolve) => {
     readyResolve = resolve;
-    readyReject = reject;
   });
 
-  const start = async () => {
+  const runInitSteps = async () => {
     const siteName = getSiteName();
     if (!siteName) {
       throw new Error("Cannot initialize deps: site name not set");
     }
     let start = performance.now();
-    try {
-      await ensureGit({
-        site: siteName,
-        repoUrl: opts?.repoUrl,
-        branch: opts?.branch,
-      });
-      readyResolve();
-    } catch (err) {
-      readyReject(err);
-      throw err;
-    }
+    await ensureGit({
+      site: siteName,
+      repoUrl: opts?.repoUrl,
+      branch: opts?.branch,
+    });
+    // Git is up — signal `ready` for any caller waiting just on the repo.
+    // Subsequent calls (on retry success) are no-ops since the Promise is
+    // already settled, which is fine.
+    readyResolve();
     logs.push({
       level: "info",
       message: `${colors.bold("[step 1/4]")}: Git setup took ${
@@ -352,8 +353,90 @@ const createDeps = (
     });
   };
 
+  // Retry init with exponential-ish backoff. Catches transient failures
+  // (GitHub App auth race, ephemeral network glitches) without leaving the
+  // pod stuck on a sticky 424 forever. Total retry window: ~52s across 6
+  // attempts — fast enough to recover within a single user request, slow
+  // enough not to hammer GitHub on permanent failures.
+  const RETRY_DELAYS_MS = [500, 1500, 5000, 15_000, 30_000];
+
+  // Sleep that wakes early if the surrounding signal aborts. Used between
+  // retry attempts so dispose() during a long backoff cancels promptly
+  // instead of waking up later into a torn-down sandbox.
+  const abortableSleep = (ms: number): Promise<void> => {
+    if (signal?.aborted) {
+      return Promise.reject(
+        new DOMException("Init aborted before backoff", "AbortError"),
+      );
+    }
+    return new Promise<void>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        signal?.removeEventListener("abort", onAbort);
+        resolve();
+      }, ms);
+      const onAbort = () => {
+        clearTimeout(timer);
+        reject(new DOMException("Init aborted during backoff", "AbortError"));
+      };
+      signal?.addEventListener("abort", onAbort, { once: true });
+    });
+  };
+
+  const runInitWithRetry = async () => {
+    let lastErr: unknown;
+    for (let i = 0; i <= RETRY_DELAYS_MS.length; i++) {
+      // Bail before each attempt — dispose() / undeploy may have aborted us
+      // while we were sleeping in backoff or running the previous attempt.
+      if (signal?.aborted) {
+        throw new DOMException("Init aborted", "AbortError");
+      }
+      try {
+        await runInitSteps();
+        return;
+      } catch (err) {
+        lastErr = err;
+        const next = RETRY_DELAYS_MS[i];
+        console.warn(
+          `[deps] init attempt ${i + 1}/${RETRY_DELAYS_MS.length + 1} failed${
+            next !== undefined ? `, retrying in ${next}ms` : " (giving up)"
+          }:`,
+          err,
+        );
+        if (next !== undefined) {
+          await abortableSleep(next);
+        }
+      }
+    }
+    // All retries exhausted. We only `throw` so callers (including the
+    // middleware) see this attempt's failure — but we deliberately do NOT
+    // `readyReject(lastErr)`. After the cooldown a future request can
+    // start a fresh sequence, and if that one succeeds, `readyResolve()`
+    // inside `runInitSteps` finally fires. Permanently rejecting `ready`
+    // here would strand any consumer that awaited it (e.g. the worker
+    // factory passed `deps.ready`), since Promises can only settle once.
+    throw lastErr;
+  };
+
+  // Cooldown after a fully exhausted retry sequence. Allows a fresh attempt
+  // on a future request when a transient external issue lasted longer than
+  // the retry window — without thrashing if the failure is permanent.
+  const RESET_AFTER_EXHAUSTION_MS = 60_000;
+  let exhaustedAt = 0;
+
   const ensureStarted = () => {
-    ok ||= start();
+    if (
+      ok && exhaustedAt &&
+      Date.now() - exhaustedAt > RESET_AFTER_EXHAUSTION_MS
+    ) {
+      ok = null;
+      exhaustedAt = 0;
+    }
+    if (!ok) {
+      ok = runInitWithRetry().catch((err) => {
+        exhaustedAt = Date.now();
+        throw err;
+      });
+    }
   };
 
   const middleware: MiddlewareHandler & {