add retry for 502,503,504 error codes

Author: chrishagglund-ship-it

.github/workflows/pull_request.yml

@@ -79,14 +79,14 @@ jobs:
           name: codecov-unit-node-${{ matrix.node-version }}
           fail_ci_if_error: false
 
-  # Integration tests (v5): one job at a time (max-parallel: 1) to avoid 502/503 on shared Conductor.
+  # Integration tests (v5): one shard at a time (max-parallel: 1) to avoid 502/503 on shared Conductor.
   # Sharding (--shard i/N) splits the suite so each job runs ~1/N of tests — keeps per-job under timeout.
   integration-tests:
     runs-on: ubuntu-latest
     timeout-minutes: 25
     strategy:
       fail-fast: false
-      max-parallel: 2
+      max-parallel: 1
       matrix:
         node-version: [20, 22, 24]
         shard: [1, 2, 3]

src/sdk/createConductorClient/helpers/__tests__/fetchWithRetry.test.ts

@@ -181,6 +181,80 @@ describe("fetchWithRetry", () => {
     });
   });
 
+  // ─── Server error (502/503/504) retry ───────────────────────────────
+
+  describe("server error (502/503/504) retry", () => {
+    it("should retry 502 and succeed on next attempt", async () => {
+      mockFetch
+        .mockResolvedValueOnce(createMockResponse(502, "Bad Gateway"))
+        .mockResolvedValueOnce(createMockResponse(200, "ok"));
+
+      const result = await retryFetch("http://test.com", {}, mockFetch, {
+        maxTransportRetries: 3,
+        initialRetryDelay: 1,
+      });
+
+      expect(result.status).toBe(200);
+      expect(mockFetch).toHaveBeenCalledTimes(2);
+    });
+
+    it("should retry 503 and succeed on next attempt", async () => {
+      mockFetch
+        .mockResolvedValueOnce(createMockResponse(503, "Service Unavailable"))
+        .mockResolvedValueOnce(createMockResponse(200, "ok"));
+
+      const result = await retryFetch("http://test.com", {}, mockFetch, {
+        maxTransportRetries: 3,
+        initialRetryDelay: 1,
+      });
+
+      expect(result.status).toBe(200);
+      expect(mockFetch).toHaveBeenCalledTimes(2);
+    });
+
+    it("should retry 504 and succeed on next attempt", async () => {
+      mockFetch
+        .mockResolvedValueOnce(createMockResponse(504, "Gateway Timeout"))
+        .mockResolvedValueOnce(createMockResponse(200, "ok"));
+
+      const result = await retryFetch("http://test.com", {}, mockFetch, {
+        maxTransportRetries: 3,
+        initialRetryDelay: 1,
+      });
+
+      expect(result.status).toBe(200);
+      expect(mockFetch).toHaveBeenCalledTimes(2);
+    });
+
+    it("should exhaust retries and return last 5xx response", async () => {
+      mockFetch.mockResolvedValue(createMockResponse(502, "Bad Gateway"));
+
+      const result = await retryFetch("http://test.com", {}, mockFetch, {
+        maxTransportRetries: 2,
+        initialRetryDelay: 1,
+      });
+
+      expect(result.status).toBe(502);
+      // 1 initial + 2 retries = 3
+      expect(mockFetch).toHaveBeenCalledTimes(3);
+    });
+
+    it("should handle transport error then 502 then success", async () => {
+      mockFetch
+        .mockRejectedValueOnce(new Error("ECONNRESET"))
+        .mockResolvedValueOnce(createMockResponse(502, "Bad Gateway"))
+        .mockResolvedValueOnce(createMockResponse(200, "ok"));
+
+      const result = await retryFetch("http://test.com", {}, mockFetch, {
+        maxTransportRetries: 3,
+        initialRetryDelay: 1,
+      });
+
+      expect(result.status).toBe(200);
+      expect(mockFetch).toHaveBeenCalledTimes(3);
+    });
+  });
+
   // ─── Auth failure (401/403) retry ──────────────────────────────────
 
   describe("auth failure (401/403) retry", () => {

src/sdk/createConductorClient/helpers/fetchWithRetry.ts

@@ -157,6 +157,18 @@ export const retryFetch = async (
       return rateLimitResponse;
     }
 
+    // Gateway error retry (502, 503, 504) -- transient proxy/server errors
+    if (response.status >= 502 && response.status <= 504) {
+      lastError = new Error(`Server error: HTTP ${response.status}`);
+      if (transportAttempt < maxTransportRetries) {
+        await new Promise((resolve) =>
+          setTimeout(resolve, withJitter(initialRetryDelay * (transportAttempt + 1)))
+        );
+        continue;
+      }
+      return response;
+    }
+
     // Auth failure retry (401/403) - only refresh+retry when the error is a token
     // problem (EXPIRED_TOKEN or INVALID_TOKEN). Permission errors should propagate
     // immediately without wasting a token refresh + retry round-trip.