Playwright e2e: caching MITM proxy with cross-shard merge

.circleci/config.yml

@@ -361,6 +361,14 @@ jobs:
       # Optional grep filter for targeted smoke runs. Empty by default →
       # whole suite runs. Set to e.g. 'patient-screenshot' to narrow.
       PLAYWRIGHT_GREP: ''
+      # Cache repeat requests to *.cbioportal.org through a local
+      # mitmdump backed by a SQLite file. restore_cache (below) seeds
+      # the file from a previous workflow run; the merge_playwright_cache
+      # job unions all 12 shards' contributions back into one for the
+      # next workflow. See end-to-end-test-playwright/proxy/cache_addon.py
+      # for the addon and proxy/merge_cache.py for the union step.
+      PW_CACHE_PROXY: '1'
+      PW_CACHE_DB: /tmp/pw-cache/cache.sqlite
     steps:
       - attach_workspace:
           at: /tmp/repo
@@ -390,6 +398,17 @@ jobs:
           # --frozen-lockfile: fail if pnpm-lock.yaml would change,
           # the pnpm equivalent of `npm ci`.
           command: pnpm install --frozen-lockfile --ignore-workspace
+      # Restore the response-cache SQLite file (if any prior workflow
+      # run produced one). The `keys:` list is tried in order; the
+      # first matching cache is used. We prefer branch-local caches
+      # because feature branches may hit different URLs than master,
+      # but fall back to master and finally to any v1 cache.
+      - restore_cache:
+          name: Restore playwright response cache
+          keys:
+            - playwright-cache-v1-{{ .Branch }}-
+            - playwright-cache-v1-master-
+            - playwright-cache-v1-
       - run:
           name: Start frontend dev server (pnpm serveDist) on localhost:3000
           background: true
@@ -429,14 +448,24 @@ jobs:
                 GREP_ARG="--grep=$PLAYWRIGHT_GREP"
                 echo "Filtering with --grep=$PLAYWRIGHT_GREP"
             fi
+            # When PW_CACHE_PROXY=1, route the playwright invocation
+            # through scripts/run-with-cache-proxy.sh, which spins up a
+            # local mitmdump that caches *.cbioportal.org responses for
+            # the lifetime of this shard. Cache is per-process — each
+            # shard is its own container and starts with an empty cache.
+            if [ "${PW_CACHE_PROXY:-0}" = "1" ]; then
+              PW_RUNNER=(./scripts/run-with-cache-proxy.sh)
+            else
+              PW_RUNNER=(pnpm exec playwright test)
+            fi
             set +e
             # --workers=3 on resource_class large (4 vCPU / 8 GB) leaves
             # one vCPU for the runner / serveDist while running 3 chromium
             # contexts. workers=4 was tried and ran slower than 3
             # (CPU-bound contention with serveDist + runner sharing the
             # 4th core). Only describes opted into mode: 'parallel'
             # actually use the extra workers; the rest still serialize.
-            pnpm exec playwright test --workers=3 \
+            "${PW_RUNNER[@]}" --workers=3 \
               --shard="${SHARD_NUM}/${CIRCLE_NODE_TOTAL}" \
               --reporter=list,junit,blob \
               $GREP_ARG
@@ -458,6 +487,26 @@ jobs:
               echo "playwright exit: 0 (this shard passed)"
             fi
             exit 0
+      # Persist this shard's cache.sqlite under a per-shard filename so
+      # the merge_playwright_cache job can union them all. The
+      # `exit 0` above means this step runs regardless of test
+      # pass/fail. cp is guarded in case the cache file is missing
+      # — e.g. when a shard crashes before producing any response.
+      - run:
+          name: Stage shard cache for workspace
+          command: |
+            mkdir -p /tmp/pw-cache-shards
+            if [ -f /tmp/pw-cache/cache.sqlite ]; then
+              cp /tmp/pw-cache/cache.sqlite \
+                 "/tmp/pw-cache-shards/cache-${CIRCLE_NODE_INDEX}.sqlite"
+              ls -la /tmp/pw-cache-shards/
+            else
+              echo "No /tmp/pw-cache/cache.sqlite to stage."
+            fi
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - pw-cache-shards
       - store_test_results:
           path: /tmp/repo/cbioportal-frontend/end-to-end-test-playwright/test-results
       - store_artifacts:
@@ -722,6 +771,50 @@ jobs:
       - store_test_results:
           path: /tmp/repo/cbioportal-frontend/end-to-end-test-playwright/test-results
 
+  # Unions the 12 per-shard cache.sqlite files produced by
+  # remote_e2e_shards into a single file and persists it via save_cache
+  # under a branch-scoped key. The shards job's restore_cache picks the
+  # most recent matching key on the next workflow run, so each run
+  # starts hot with the union of every shard's previous contributions.
+  #
+  # Wired with `requires: [remote_e2e_shards]` (not the API-polling
+  # pattern remote_e2e uses) on purpose: if any shard is red, the
+  # workflow's tests are broken and we don't want to expand the cache
+  # with potentially-wrong responses. The next green run will refresh
+  # it. Skipping cache updates on red runs is a feature.
+  merge_playwright_cache:
+    docker:
+      - image: ghcr.io/cbioportal/cbioportal-frontend-playwright-ci:v1.59.1-jammy
+    resource_class: small
+    working_directory: /tmp/repo
+    environment:
+      HOME: /tmp
+    steps:
+      # Source comes via the workspace persisted by build_frontend; the
+      # shards job adds pw-cache-shards/ to the same workspace tree.
+      - attach_workspace:
+          at: /tmp/repo
+      - run:
+          name: Merge per-shard cache.sqlite files
+          command: |
+            mkdir -p /tmp/pw-cache
+            python3 cbioportal-frontend/end-to-end-test-playwright/proxy/merge_cache.py \
+              /tmp/repo/pw-cache-shards \
+              /tmp/pw-cache/cache.sqlite
+            ls -la /tmp/pw-cache/
+      - save_cache:
+          name: Save merged playwright cache
+          # epoch in the key makes each run write a fresh entry rather
+          # than colliding with the existing key (which save_cache would
+          # silently no-op). restore_cache uses the prefix and picks
+          # the most recent, so older entries naturally age out.
+          key: playwright-cache-v1-{{ .Branch }}-{{ epoch }}
+          paths:
+            - /tmp/pw-cache/cache.sqlite
+      - store_artifacts:
+          path: /tmp/pw-cache/cache.sqlite
+          destination: cache.sqlite
+
   analyze_flakes:
     # Read-only post-step that surfaces tests whose pass/fail outcome
     # flips across recent CI runs ("cross-run flakes" — distinct from
@@ -1593,6 +1686,16 @@ workflows:
       # uses CircleCI's public workflow API (no auth), so this works on
       # fork PRs too.
       - remote_e2e
+      # Unions all 12 shard caches into one file and save_caches it for
+      # the next workflow run. Requires green shards (see comment on
+      # the job for why).
+      - merge_playwright_cache:
+          requires:
+            # Need build_frontend so the workspace carries the source
+            # (merge_cache.py). And remote_e2e_shards for the per-shard
+            # cache.sqlite files.
+            - build_frontend
+            - remote_e2e_shards
       # Read-only post-step on remote_e2e: detects cross-run flakes
       # (pass in some PRs, fail in others) and writes a Claude-authored
       # analysis as a build artifact. Runs after remote_e2e so this

.circleci/images/playwright/Dockerfile

@@ -19,10 +19,20 @@
 
 FROM mcr.microsoft.com/playwright:v1.59.1-jammy
 
-# Install jq (used by scripts/env_vars.sh to parse the GitHub API response when
-# resolving the PR's base branch).
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends jq \
+# Install jq (used by scripts/env_vars.sh to parse the GitHub API response
+# when resolving the PR's base branch) and mitmproxy via pip (used by
+# end-to-end-test-playwright/scripts/run-with-cache-proxy.sh to cache repeat
+# requests to *.cbioportal.org during a single test run; see that script's
+# header for the wiring).
+#
+# Why pip rather than apt: Ubuntu jammy ships mitmproxy 6.0.2, which has
+# unreliable HTTP/2 + addon-hook firing on modern Chromium TLS. The pip
+# release (11.x at time of writing) has a working request/response hook
+# pipeline. DEBIAN_FRONTEND=noninteractive prevents tzdata (pulled in
+# transitively by python3-pip) from blocking on its interactive prompt.
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends jq python3-pip \
+    && pip3 install --no-cache-dir 'mitmproxy>=11,<12' \
     && rm -rf /var/lib/apt/lists/*
 
 # Enable corepack so the `pnpm` shim is on PATH, then pre-fetch and activate

end-to-end-test-playwright/playwright.config.ts

@@ -41,6 +41,18 @@ const updateSnapshots = process.env.PW_UPDATE_SNAPSHOTS as
 // 5+ minutes to slow shards. The localdb job opts in via PW_LOCAL=1.
 const includeLocalDb = process.env.PW_LOCAL === '1';
 
+// When scripts/run-with-cache-proxy.sh is in play, HTTPS_PROXY points
+// at a local mitmdump that caches *.cbioportal.org responses for the
+// duration of a single test run. Routing Playwright's browser through
+// it requires (a) the proxy server setting and (b) accepting the
+// proxy's self-signed CA — easier than installing the CA into Chromium.
+const proxyServer = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
+const proxy = proxyServer ? { server: proxyServer } : undefined;
+if (proxy) {
+    // eslint-disable-next-line no-console
+    console.log(`[playwright.config] routing through proxy ${proxy.server}`);
+}
+
 export default defineConfig({
     testDir: './tests',
     testIgnore: includeLocalDb ? [] : ['**/local/**'],
@@ -80,7 +92,8 @@ export default defineConfig({
         video: 'retain-on-failure',
         actionTimeout: 15_000,
         navigationTimeout: 60_000,
-        ...(isLocaldev && { ignoreHTTPSErrors: true }),
+        ...((isLocaldev || proxy) && { ignoreHTTPSErrors: true }),
+        ...(proxy && { proxy }),
     },
 
     projects: [
@@ -106,6 +119,24 @@ export default defineConfig({
                         '--disable-font-subpixel-positioning',
                         '--disable-lcd-text',
                         '--font-render-hinting=none',
+                        // When routed through scripts/run-with-cache-proxy.sh,
+                        // mitmdump presents a self-signed cert generated on
+                        // first use. Chromium gates proxy/MITM certs *before*
+                        // Playwright's context-level ignoreHTTPSErrors gets a
+                        // chance to respond to the CDP request, so the only
+                        // reliable way to make TLS interception work is the
+                        // launch-level --ignore-certificate-errors flag.
+                        // We also pass --proxy-server here in addition to
+                        // use.proxy: in chromium-headless-shell builds the
+                        // CDP-level proxy setting (use.proxy) was observed to
+                        // silently no-op for HTTPS traffic, while the
+                        // launch-level switch is honoured reliably.
+                        ...(proxy
+                            ? [
+                                  '--ignore-certificate-errors',
+                                  `--proxy-server=${proxy.server}`,
+                              ]
+                            : []),
                         ...(isLocaldev
                             ? [
                                   // Private Network Access + the newer

end-to-end-test-playwright/proxy/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

end-to-end-test-playwright/proxy/Dockerfile.overlay

@@ -0,0 +1,27 @@
+# Thin overlay used only for local development while the canonical CI
+# image (.circleci/images/playwright/Dockerfile) is being rebuilt with
+# mitmproxy baked in. Build once:
+#
+#   docker build --platform linux/amd64 \
+#     -f end-to-end-test-playwright/proxy/Dockerfile.overlay \
+#     -t cbioportal-frontend-playwright-ci:cache-local \
+#     end-to-end-test-playwright/proxy
+#
+# Then run the suite via:
+#
+#   PW_CACHE_PROXY=1 \
+#   PW_CACHE_IMAGE=cbioportal-frontend-playwright-ci:cache-local \
+#   ./scripts/docker-test.sh tests/your-spec.ts
+#
+# Once the GHCR image is republished, drop this overlay and the
+# PW_CACHE_IMAGE override.
+FROM ghcr.io/cbioportal/cbioportal-frontend-playwright-ci:v1.59.1-jammy
+USER root
+# Ubuntu jammy's apt mitmproxy is 6.0.2, which has known HTTP/2 +
+# addon-hook reliability issues. pip pulls a current release (11.x at
+# time of writing) with a working request/response hook pipeline. Pin
+# loosely so security backports flow in.
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-pip \
+    && pip3 install --no-cache-dir 'mitmproxy>=11,<12' \
+    && rm -rf /var/lib/apt/lists/*