fix(temporal): retry Glitter startup metric restoration

packages/docs/plans/2026-08-07_glitter-corpus-startup-metric-retry.md

@@ -0,0 +1,35 @@
+---
+id: plan-2026-08-07-glitter-corpus-startup-metric-retry
+type: plan
+status: in-progress
+board: false
+---
+
+# Self-healing Glitter startup metric restoration
+
+## Summary
+
+Add an indefinite, exponential-backoff retry supervisor for Glitter's startup
+snapshot-metric restoration. This prevents transient SeaweedFS startup races
+from producing a false stale alert while preserving the existing alert for
+genuinely missing, invalid, or inaccessible snapshots.
+
+## Implementation
+
+- Add an injectable startup retry helper with equal-jitter exponential backoff:
+  10-second initial ceiling, doubling per attempt, capped at five minutes.
+- Retry only transient SeaweedFS connection and 408/429/5xx failures. Do not
+  retry missing pointers, authorization failures, malformed data, checksum
+  failures, or schema violations.
+- Stop retrying when worker shutdown begins, log every retry, and emit one
+  Sentry warning after ten consecutive transient failures.
+- Keep the existing Prometheus alert expression, schedule timing, and
+  PagerDuty routing unchanged.
+
+## Verification
+
+- Cover retry success, jitter bounds, cap, shutdown cancellation, persistent
+  transient failures, and storage-error classification with `bun:test`.
+- Run the Temporal package tests, typecheck, and lint.
+- After deployment, verify startup restoration, the snapshot timestamp metric,
+  alert resolution, and natural PagerDuty resolution.

packages/temporal/src/activities/glitter-corpus-storage.test.ts

@@ -3,7 +3,10 @@ import { describe, expect, spyOn, test } from "bun:test";
 import { glitterCorpusStorageIntegrityFailuresTotal } from "#observability/metrics-glitter.ts";
 import { StoredObjectSchema } from "#shared/glitter-corpus.ts";
 import { discordRequestLeaseDelayMs } from "./glitter-corpus-rate-limit.ts";
-import type { CorpusStore } from "./glitter-corpus-store.ts";
+import {
+  isTransientCorpusStorageError,
+  type CorpusStore,
+} from "./glitter-corpus-store.ts";
 import {
   LatestSnapshotPointerSchema,
   latestSnapshotPointerNeedsUpdate,
@@ -57,6 +60,41 @@ describe("Glitter corpus latest snapshot pointer", () => {
   });
 });
 
+describe("Glitter corpus transient storage errors", () => {
+  test("recognizes transient connection failures", () => {
+    for (const code of [
+      "ECONNREFUSED",
+      "ECONNRESET",
+      "ETIMEDOUT",
+      "EAI_AGAIN",
+      "ENOTFOUND",
+    ]) {
+      expect(isTransientCorpusStorageError(new Error(code))).toBe(true);
+    }
+  });
+
+  test("recognizes retryable HTTP responses", () => {
+    for (const statusCode of [408, 429, 500, 503]) {
+      const error = Object.assign(new Error(`HTTP ${String(statusCode)}`), {
+        $metadata: { httpStatusCode: statusCode },
+      });
+      expect(isTransientCorpusStorageError(error)).toBe(true);
+    }
+  });
+
+  test("does not retry permanent storage failures", () => {
+    for (const statusCode of [401, 403, 404]) {
+      const error = Object.assign(new Error(`HTTP ${String(statusCode)}`), {
+        $metadata: { httpStatusCode: statusCode },
+      });
+      expect(isTransientCorpusStorageError(error)).toBe(false);
+    }
+    expect(
+      isTransientCorpusStorageError(new Error("invalid snapshot JSON")),
+    ).toBe(false);
+  });
+});
+
 describe("Glitter corpus stored object receipt", () => {
   test("requires one matching SeaweedFS receipt", () => {
     const stored = StoredObjectSchema.parse({

packages/temporal/src/activities/glitter-corpus-store.ts

@@ -20,6 +20,26 @@ const S3ErrorShapeSchema = z.object({
   $metadata: z.object({ httpStatusCode: z.number().optional() }).optional(),
 });
 
+const TRANSIENT_STORAGE_ERROR_PATTERN =
+  /\b(?:ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND)\b/i;
+
+export function isTransientCorpusStorageError(error: unknown): boolean {
+  const parsed = S3ErrorShapeSchema.safeParse(error);
+  const statusCode = parsed.success
+    ? parsed.data.$metadata?.httpStatusCode
+    : undefined;
+  if (
+    statusCode !== undefined &&
+    (statusCode === 408 || statusCode === 429 || statusCode >= 500)
+  ) {
+    return true;
+  }
+  return (
+    error instanceof Error &&
+    TRANSIENT_STORAGE_ERROR_PATTERN.test(`${error.name} ${error.message}`)
+  );
+}
+
 export function isNotFoundError(error: unknown): boolean {
   const parsed = S3ErrorShapeSchema.safeParse(error);
   if (!parsed.success) {

packages/temporal/src/shared/startup-retry.test.ts

@@ -0,0 +1,124 @@
+import { describe, expect, test } from "bun:test";
+import {
+  equalJitterRetryDelayMs,
+  retryUntilReady,
+  STARTUP_RETRY_MAXIMUM_DELAY_MS,
+} from "./startup-retry.ts";
+
+describe("equal-jitter startup retry delays", () => {
+  test("doubles the ceiling and keeps delays within the equal-jitter range", () => {
+    expect(equalJitterRetryDelayMs(1, 0)).toBe(5000);
+    expect(equalJitterRetryDelayMs(1, 1)).toBe(10_000);
+    expect(equalJitterRetryDelayMs(2, 0.5)).toBe(15_000);
+  });
+
+  test("caps the exponential ceiling at five minutes", () => {
+    expect(equalJitterRetryDelayMs(20, 0)).toBe(150_000);
+    expect(equalJitterRetryDelayMs(20, 1)).toBe(STARTUP_RETRY_MAXIMUM_DELAY_MS);
+  });
+});
+
+describe("retryUntilReady", () => {
+  test("retries a transient operation until it succeeds", async () => {
+    let calls = 0;
+    const delays: number[] = [];
+
+    const result = await retryUntilReady({
+      operation: async () => {
+        calls += 1;
+        if (calls < 3) {
+          throw new Error("ECONNREFUSED");
+        }
+      },
+      shouldRetry: () => true,
+      isClosed: () => false,
+      random: () => 0,
+      sleep: async (delayMs) => {
+        delays.push(delayMs);
+      },
+    });
+
+    expect(result).toBe("succeeded");
+    expect(calls).toBe(3);
+    expect(delays).toEqual([5000, 10_000]);
+  });
+
+  test("does not retry a non-transient failure", async () => {
+    let calls = 0;
+    let sleeps = 0;
+
+    await expect(
+      retryUntilReady({
+        operation: async () => {
+          calls += 1;
+          throw new Error("access denied");
+        },
+        shouldRetry: () => false,
+        isClosed: () => false,
+        sleep: async () => {
+          sleeps += 1;
+        },
+      }),
+    ).rejects.toThrow("access denied");
+
+    expect(calls).toBe(1);
+    expect(sleeps).toBe(0);
+  });
+
+  test("stops after shutdown during a retry delay", async () => {
+    let calls = 0;
+    let closed = false;
+
+    const result = await retryUntilReady({
+      operation: async () => {
+        calls += 1;
+        throw new Error("ETIMEDOUT");
+      },
+      shouldRetry: () => true,
+      isClosed: () => closed,
+      random: () => 0,
+      sleep: async () => {
+        closed = true;
+      },
+    });
+
+    expect(result).toBe("closed");
+    expect(calls).toBe(1);
+  });
+
+  test("continues transient retries and escalates once", async () => {
+    let calls = 0;
+    let closed = false;
+    const retries: number[] = [];
+    const escalations: number[] = [];
+
+    const result = await retryUntilReady({
+      operation: async () => {
+        calls += 1;
+        throw new Error("HTTP 503");
+      },
+      shouldRetry: () => true,
+      isClosed: () => closed,
+      random: () => 0,
+      onRetry: ({ attempt }) => {
+        retries.push(attempt);
+      },
+      onEscalate: ({ attempt }) => {
+        escalations.push(attempt);
+      },
+      sleep: async (_delayMs, isClosed) => {
+        if (retries.length === 11) {
+          closed = true;
+        }
+        expect(isClosed()).toBe(closed);
+      },
+    });
+
+    expect(result).toBe("closed");
+    expect(calls).toBe(11);
+    expect(retries).toEqual(
+      Array.from({ length: 11 }, (_, index) => index + 1),
+    );
+    expect(escalations).toEqual([10]);
+  });
+});

packages/temporal/src/shared/startup-retry.ts

@@ -0,0 +1,112 @@
+export const STARTUP_RETRY_INITIAL_DELAY_MS = 10_000;
+export const STARTUP_RETRY_MAXIMUM_DELAY_MS = 300_000;
+export const STARTUP_RETRY_ESCALATION_ATTEMPT = 10;
+
+export type RetrySleep = (
+  delayMs: number,
+  isClosed: () => boolean,
+) => Promise<void>;
+
+export type StartupRetryFailure = {
+  readonly attempt: number;
+  readonly delayMs: number;
+  readonly error: unknown;
+};
+
+export type StartupRetryInput = {
+  readonly operation: () => Promise<void>;
+  readonly shouldRetry: (error: unknown) => boolean;
+  readonly isClosed: () => boolean;
+  readonly sleep?: RetrySleep;
+  readonly random?: () => number;
+  readonly initialDelayMs?: number;
+  readonly maximumDelayMs?: number;
+  readonly onRetry?: (failure: StartupRetryFailure) => void;
+  readonly onEscalate?: (failure: StartupRetryFailure) => void;
+};
+
+export type StartupRetryResult = "succeeded" | "closed";
+
+export function equalJitterRetryDelayMs(
+  attempt: number,
+  randomValue: number,
+  initialDelayMs = STARTUP_RETRY_INITIAL_DELAY_MS,
+  maximumDelayMs = STARTUP_RETRY_MAXIMUM_DELAY_MS,
+): number {
+  if (!Number.isSafeInteger(attempt) || attempt < 1) {
+    throw new Error(
+      `retry attempt must be a positive integer, got ${String(attempt)}`,
+    );
+  }
+  if (randomValue < 0 || randomValue > 1 || !Number.isFinite(randomValue)) {
+    throw new Error(
+      `retry random value must be between 0 and 1, got ${String(randomValue)}`,
+    );
+  }
+  if (initialDelayMs <= 0 || maximumDelayMs < initialDelayMs) {
+    throw new Error("retry delay bounds are invalid");
+  }
+
+  const exponentialDelay = Math.min(
+    maximumDelayMs,
+    initialDelayMs * 2 ** (attempt - 1),
+  );
+  return Math.round(
+    exponentialDelay / 2 + randomValue * (exponentialDelay / 2),
+  );
+}
+
+export async function sleepUnlessClosed(
+  delayMs: number,
+  isClosed: () => boolean,
+  sleep: (delayMs: number) => Promise<void> = Bun.sleep,
+): Promise<void> {
+  const deadline = Date.now() + delayMs;
+  while (!isClosed() && Date.now() < deadline) {
+    const remainingMs = deadline - Date.now();
+    await sleep(Math.min(remainingMs, 1000));
+  }
+}
+
+export async function retryUntilReady(
+  input: StartupRetryInput,
+): Promise<StartupRetryResult> {
+  const sleep: RetrySleep =
+    input.sleep ??
+    ((delayMs, isClosed) => sleepUnlessClosed(delayMs, isClosed));
+  const random = input.random ?? Math.random;
+  let attempt = 0;
+
+  while (!input.isClosed()) {
+    try {
+      await input.operation();
+      return "succeeded";
+    } catch (error: unknown) {
+      if (input.isClosed()) {
+        return "closed";
+      }
+      if (!input.shouldRetry(error)) {
+        throw error;
+      }
+
+      attempt += 1;
+      const failure: StartupRetryFailure = {
+        attempt,
+        delayMs: equalJitterRetryDelayMs(
+          attempt,
+          random(),
+          input.initialDelayMs,
+          input.maximumDelayMs,
+        ),
+        error,
+      };
+      input.onRetry?.(failure);
+      if (attempt === STARTUP_RETRY_ESCALATION_ATTEMPT) {
+        input.onEscalate?.(failure);
+      }
+      await sleep(failure.delayMs, input.isClosed);
+    }
+  }
+
+  return "closed";
+}