Make all GitHub Actions cache service HTTP errors non-fatal, and add scheduled live test (#1306)

HTTP errors from the GitHub Actions cache service are no longer ever fatal, and should always continue gracefully with GitHub Actions Caching temporarily disabled. Previously only network errors and HTTP 429/503 errors failed gracefully.

This should mean that if something like #1297 happens again (where we started getting 422 "Unprocessable" errors because of a breaking change to their API), users won't be broken.

Also adds a scheduled daily test of the live service (just one platform, morning PT), since the above change would otherwise reduce the timeliness of us finding out about a problem with the production service.

Author: aomarks

.github/workflows/test-real-github-actions-cache-service.yml

@@ -0,0 +1,33 @@
+name: Test real GitHub Actions cache service
+
+on:
+  schedule:
+    - cron: '14 7 * * *' # 6:07AM PST / 7:07AM PDT
+
+jobs:
+  tests:
+    strategy:
+      matrix:
+        include:
+          - node: 20 # LTS
+            os: ubuntu-22.04
+
+    runs-on: ${{ matrix.os }}
+
+    env:
+      WIREIT_LOGGER: 'quiet-ci'
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: npm
+
+      # Note we intentionally don't enable Wireit caching for this workflow,
+      # because we're looking for problems coming from the real production
+      # GitHub Actions cache service, which is independent of any of our files.
+      # - uses: google/wireit@setup-github-actions-caching/v2
+
+      - run: npm ci
+      - run: npm run test:cache-github-real

CHANGELOG.md

@@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic
 Versioning](https://semver.org/spec/v2.0.0.html).
 
-<!-- ## Unreleased -->
+## Unreleased
+
+- HTTP errors from the GitHub Actions cache service are no longer ever fatal,
+  and should always continue gracefully with GitHub Actions caching temporarily
+  disabled (previously only network errors and HTTP 429/503 errors failed
+  gracefully).
 
 ## [0.14.12] - 2025-04-10
 

src/caching/github-actions-cache.ts

@@ -27,7 +27,7 @@ import type {Result} from '../error.js';
 import type {InvalidUsage, UnknownErrorThrown} from '../event.js';
 
 /**
- * Caches script output to the GitHub Actions caching service.
+ * Caches script output to the GitHub Actions cache service.
  */
 export class GitHubActionsCache implements Cache {
   readonly #baseUrl: string;
@@ -504,59 +504,66 @@ ${blockIds.map((blockId) => `  <Uncommitted>${blockId}</Uncommitted>`).join('\n'
   }
 
   /**
-   * If we received an error that indicates something is wrong with the GitHub
-   * Actions service that is not our fault, log an error and return false.
-   * Otherwise return true.
+   * If we received a network or HTTP error from the given HTTP request, set the
+   * global flag to indicate that GitHub Actions Caching is down (it's a class
+   * property, but this class should be a global singleton),  log an error
+   * (possibly asynchronously), and return false. If the request was OK, just
+   * return true.
    */
   #maybeHandleServiceDown(
     res: Result<http.IncomingMessage, Error>,
     script: ScriptReference,
   ): res is {ok: true; value: http.IncomingMessage} {
+    const status = res.ok ? res.value.statusCode : null;
+    if (res.ok && status != null && status >= 200 && status <= 299) {
+      return true;
+    }
+
+    if (this.#serviceIsDown) {
+      // Reduce noise; just whatever the first error was is OK.
+      //
+      // Note that this cache can be accessed concurrently, so even though we
+      // stop making HTTP requests after setting this flag, there could be >1
+      // pending requests out at the same time before the first error is
+      // detected.
+      return false;
+    }
+    this.#serviceIsDown = true;
+
     if (!res.ok) {
-      if (!this.#serviceIsDown) {
+      this.#logger.log({
+        script,
+        type: 'info',
+        detail: 'cache-info',
+        message:
+          `Network error from GitHub Actions cache service.` +
+          ` GitHub Actions caching has been temporarily disabled.` +
+          ` Detail:\n\n${res.error}`,
+      });
+    } else {
+      void (async () => {
+        const body = await readBody(res.value).catch(() => '');
+        if (this.#serviceIsDown) {
+          return;
+        }
+        const message =
+          status === 429
+            ? `Hit GitHub Actions cache service rate limit`
+            : status === 503
+              ? `GitHub Actions cache service is temporarily unavailable`
+              : `Unexpected HTTP ${status} error from GitHub Actions cache service`;
         this.#logger.log({
           script,
           type: 'info',
           detail: 'cache-info',
           message:
-            `Connection error from GitHub Actions service, caching disabled. ` +
-            'Detail: ' +
-            ('code' in res.error
-              ? `${(res.error as Error & {code: string}).code} `
-              : '') +
-            res.error.message,
+            `${message}.` +
+            ` GitHub Actions caching has been temporarily disabled.` +
+            ` Detail:\n\nHTTP ${status}: ${body}`,
         });
-      }
-    } else {
-      switch (res.value.statusCode) {
-        case /* Too Many Requests */ 429: {
-          if (!this.#serviceIsDown) {
-            this.#logger.log({
-              script,
-              type: 'info',
-              detail: 'cache-info',
-              message: `Hit GitHub Actions cache rate limit, caching disabled.`,
-            });
-          }
-          break;
-        }
-        case /* Service Unavailable */ 503: {
-          if (!this.#serviceIsDown) {
-            this.#logger.log({
-              script,
-              type: 'info',
-              detail: 'cache-info',
-              message: `GitHub Actions service is unavailable, caching disabled.`,
-            });
-          }
-          break;
-        }
-        default: {
-          return true;
-        }
-      }
+      })();
     }
-    this.#serviceIsDown = true;
+
     return false;
   }
 
@@ -678,6 +685,11 @@ ${blockIds.map((blockId) => `  <Uncommitted>${blockId}</Uncommitted>`).join('\n'
     req.end(bodyBuffer);
 
     const result = await resPromise;
+    if (result.ok && result.value.statusCode === /* Conflict */ 409) {
+      // We must have been racing with another concurrent process running the
+      // same script.
+      return undefined;
+    }
     if (!this.#maybeHandleServiceDown(result, script)) {
       return undefined;
     }
@@ -690,10 +702,6 @@ ${blockIds.map((blockId) => `  <Uncommitted>${blockId}</Uncommitted>`).join('\n'
       return resData.signed_upload_url;
     }
 
-    if (response.statusCode === /* Conflict */ 409) {
-      return undefined;
-    }
-
     throw new Error(
       `GitHub Cache reserve HTTP ${String(
         response.statusCode,

src/test/cache-github-fake.test.ts

@@ -234,9 +234,118 @@ function assertSuccess(exitResult: ExitResult) {
   }
 }
 
-for (const code of [429, 503, 'ECONNRESET'] as const) {
+test(
+  `gracefully handles 409 conflict on set()`,
+  rigTest(async ({rig, server}) => {
+    const cmdA = await rig.newCommand();
+    const cmdB = await rig.newCommand();
+    await rig.write({
+      'package.json': {
+        scripts: {
+          a: 'wireit',
+          b: 'wireit',
+        },
+        wireit: {
+          a: {
+            command: cmdA.command,
+            files: ['input'],
+            output: [],
+            dependencies: ['b'],
+          },
+          b: {
+            command: cmdB.command,
+            files: ['input'],
+            output: [],
+          },
+        },
+      },
+      input: 'foo',
+    });
+
+    server.forceErrorOnNextRequest('createCacheEntry', 409);
+    const exec = rig.exec('npm run a');
+    (await cmdB.nextInvocation()).exit(0);
+    (await cmdA.nextInvocation()).exit(0);
+    assertSuccess(await exec.exit);
+    assert.equal(server.metrics, {
+      // Both scripts should check for and then try to reserve a cache entry ...
+      getCacheEntry: 2,
+      createCacheEntry: 2,
+      // ... but since the first one will fail to reserve with a 409, only the
+      // second one will actually finish uploading its entry.
+      putBlobBlock: 1,
+      putBlobBlockList: 1,
+      finalizeCacheEntry: 1,
+      getBlob: 0,
+    } satisfies FakeGitHubActionsCacheServerMetrics);
+    assert.equal(cmdA.numInvocations, 1);
+    assert.equal(cmdA.numInvocations, 1);
+  }),
+);
+
+const randomInt = (minIncl: number, maxExcl: number) =>
+  minIncl + Math.floor(Math.random() * (maxExcl - minIncl));
+
+for (const code of [
+  'ECONNRESET',
+  (() => {
+    while (true) {
+      const status = randomInt(400, 500);
+      if (status !== /* Conflict has special meaning (see above test) */ 409) {
+        return status;
+      }
+    }
+  })(),
+  randomInt(500, 600),
+] as const) {
+  test(
+    `recovers from ${code} error within get()`,
+    rigTest(async ({rig, server}) => {
+      const cmdA = await rig.newCommand();
+      const cmdB = await rig.newCommand();
+      await rig.write({
+        'package.json': {
+          scripts: {
+            a: 'wireit',
+            b: 'wireit',
+          },
+          wireit: {
+            a: {
+              command: cmdA.command,
+              files: ['input'],
+              output: [],
+              dependencies: ['b'],
+            },
+            b: {
+              command: cmdB.command,
+              files: ['input'],
+              output: [],
+            },
+          },
+        },
+        input: 'foo',
+      });
+
+      server.forceErrorOnNextRequest('getCacheEntry', code);
+      const exec = rig.exec('npm run a');
+      (await cmdB.nextInvocation()).exit(0);
+      (await cmdA.nextInvocation()).exit(0);
+      assertSuccess(await exec.exit);
+      assert.equal(server.metrics, {
+        getCacheEntry: 1,
+        createCacheEntry: 0,
+        putBlobBlock: 0,
+        putBlobBlockList: 0,
+        finalizeCacheEntry: 0,
+        getBlob: 0,
+      } satisfies FakeGitHubActionsCacheServerMetrics);
+      assert.equal(cmdA.numInvocations, 1);
+      assert.equal(cmdA.numInvocations, 1);
+    }),
+  );
+
   test(
-    `recovers from ${code} error`,
+    `recovers from ${code} error within set()`,
     rigTest(async ({rig, server}) => {
       await rig.write({
         'package.json': {