diff --git a/daemon/main.ts b/daemon/main.ts index 3de202b2b..57e897e40 100644 --- a/daemon/main.ts +++ b/daemon/main.ts @@ -271,29 +271,30 @@ const createDeps = ( ): MiddlewareHandler & { ready: Promise; ensureStarted: () => void } => { let ok: Promise | null = null; let readyResolve: () => void; - let readyReject: (err: unknown) => void; - const ready = new Promise((resolve, reject) => { + // `ready` is intentionally never rejected. We resolve it on the first + // successful `ensureGit` (inside `runInitSteps`); failures are surfaced + // to the caller via the `ok` Promise instead. Keeping `ready` settle-once + // and resolve-only means consumers that await it (e.g. worker factories) + // can recover after a cooldown-driven retry. + const ready = new Promise((resolve) => { readyResolve = resolve; - readyReject = reject; }); - const start = async () => { + const runInitSteps = async () => { const siteName = getSiteName(); if (!siteName) { throw new Error("Cannot initialize deps: site name not set"); } let start = performance.now(); - try { - await ensureGit({ - site: siteName, - repoUrl: opts?.repoUrl, - branch: opts?.branch, - }); - readyResolve(); - } catch (err) { - readyReject(err); - throw err; - } + await ensureGit({ + site: siteName, + repoUrl: opts?.repoUrl, + branch: opts?.branch, + }); + // Git is up — signal `ready` for any caller waiting just on the repo. + // Subsequent calls (on retry success) are no-ops since the Promise is + // already settled, which is fine. + readyResolve(); logs.push({ level: "info", message: `${colors.bold("[step 1/4]")}: Git setup took ${ @@ -352,8 +353,90 @@ const createDeps = ( }); }; + // Retry init with exponential-ish backoff. Catches transient failures + // (GitHub App auth race, ephemeral network glitches) without leaving the + // pod stuck on a sticky 424 forever. Total retry window: ~52s across 6 + // attempts — fast enough to recover within a single user request, slow + // enough not to hammer GitHub on permanent failures. + const RETRY_DELAYS_MS = [500, 1500, 5000, 15_000, 30_000]; + + // Sleep that wakes early if the surrounding signal aborts. Used between + // retry attempts so dispose() during a long backoff cancels promptly + // instead of waking up later into a torn-down sandbox. + const abortableSleep = (ms: number): Promise => { + if (signal?.aborted) { + return Promise.reject( + new DOMException("Init aborted before backoff", "AbortError"), + ); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(new DOMException("Init aborted during backoff", "AbortError")); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); + }; + + const runInitWithRetry = async () => { + let lastErr: unknown; + for (let i = 0; i <= RETRY_DELAYS_MS.length; i++) { + // Bail before each attempt — dispose() / undeploy may have aborted us + // while we were sleeping in backoff or running the previous attempt. + if (signal?.aborted) { + throw new DOMException("Init aborted", "AbortError"); + } + try { + await runInitSteps(); + return; + } catch (err) { + lastErr = err; + const next = RETRY_DELAYS_MS[i]; + console.warn( + `[deps] init attempt ${i + 1}/${RETRY_DELAYS_MS.length + 1} failed${ + next !== undefined ? `, retrying in ${next}ms` : " (giving up)" + }:`, + err, + ); + if (next !== undefined) { + await abortableSleep(next); + } + } + } + // All retries exhausted. We only `throw` so callers (including the + // middleware) see this attempt's failure — but we deliberately do NOT + // `readyReject(lastErr)`. After the cooldown a future request can + // start a fresh sequence, and if that one succeeds, `readyResolve()` + // inside `runInitSteps` finally fires. Permanently rejecting `ready` + // here would strand any consumer that awaited it (e.g. the worker + // factory passed `deps.ready`), since Promises can only settle once. + throw lastErr; + }; + + // Cooldown after a fully exhausted retry sequence. Allows a fresh attempt + // on a future request when a transient external issue lasted longer than + // the retry window — without thrashing if the failure is permanent. + const RESET_AFTER_EXHAUSTION_MS = 60_000; + let exhaustedAt = 0; + const ensureStarted = () => { - ok ||= start(); + if ( + ok && exhaustedAt && + Date.now() - exhaustedAt > RESET_AFTER_EXHAUSTION_MS + ) { + ok = null; + exhaustedAt = 0; + } + if (!ok) { + ok = runInitWithRetry().catch((err) => { + exhaustedAt = Date.now(); + throw err; + }); + } }; const middleware: MiddlewareHandler & {