Stabilise visual regression snapshot testing

Unrelated PRs turned red from snapshot "changes" that were only a
couple of bytes. The Playwright config set updateSnapshots: 'all',
so every run unconditionally rewrote all 1482 PNGs and 471 ARIA
snapshots instead of comparing against the committed baseline. The
workflow then auto-committed any byte diff and hard-failed, so
non-deterministic Chromium anti-aliasing produced spurious commits
and red CI on PRs that changed no component behaviour.

Switches the model to assert-against-baseline, regenerate on intent:

- Removes updateSnapshots: 'all' so runs compare against the
  committed baseline, and adds maxDiffPixelRatio: 0.01 so trivial
  anti-aliasing noise passes instead of failing.
- Gates regeneration and the auto-commit behind a regen-snapshots
  PR label; normal runs only assert and fail loudly on a real diff.
- Forwards extra args through script/run-playwright so the regen
  path can pass --update-snapshots.
- Raises timeout-minutes from 20 to 45 and drops continue-on-error
  and the blanket exit-1 step so a partial or timed-out run fails
  instead of committing a partial baseline on 2-core runners.

Implements DREAM-773.

Author: myabc

.github/workflows/test-visual.yml

@@ -13,12 +13,18 @@ concurrency:
 
 env:
   FERRUM_PROCESS_TIMEOUT: 30
+  # Regeneration is opt-in: add the `regen-snapshots` label to a PR to rewrite
+  # and commit the baseline. Without it the job only asserts against the
+  # committed snapshots and fails loudly on a real diff — it never commits.
+  REGEN: ${{ contains(github.event.pull_request.labels.*.name, 'regen-snapshots') }}
 
 jobs:
   visual:
     name: Visual and Semantic Markup Regressions
     if: ${{ github.event_name == 'pull_request' }}
-    timeout-minutes: 20
+    # 2-core ubuntu-latest: a full suite does not finish inside 20 min. Raised
+    # so a run completes instead of timing out and committing a partial baseline.
+    timeout-minutes: 45
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
@@ -52,12 +58,20 @@ jobs:
           gem install overmind
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps
-      - name: Run Playwright tests
-        id: playwright-run
-        continue-on-error: true
+      # Default path: compare against the committed baseline. No
+      # continue-on-error — a genuine over-threshold diff fails the job so drift
+      # surfaces loudly instead of being silently auto-committed.
+      - name: Run Playwright tests (assert)
+        id: playwright-assert
+        if: ${{ env.REGEN != 'true' }}
         run: ./script/run-playwright
+      # Opt-in path (regen-snapshots label): rewrite the baseline, then commit it.
+      - name: Run Playwright tests (regenerate)
+        id: playwright-regen
+        if: ${{ env.REGEN == 'true' }}
+        run: ./script/run-playwright --update-snapshots
       - name: Update to latest branch tip
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        if: ${{ env.REGEN == 'true' && github.event.pull_request.head.repo.full_name == github.repository }}
         run: |
           set -e
           BRANCH="${GITHUB_EVENT_PULL_REQUEST_HEAD_REF}"
@@ -73,15 +87,15 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_EVENT_PULL_REQUEST_HEAD_REF: ${{ github.event.pull_request.head.ref }}
       - id: auto-commit
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        if: ${{ env.REGEN == 'true' && github.event.pull_request.head.repo.full_name == github.repository }}
         uses: stefanzweifel/git-auto-commit-action@04702edda442b2e678b25b537cec683a1493fcb9 # v7.1.0
         with:
           commit_message: Generating component snapshots
           file_pattern: .playwright/screenshots/**/*.png .playwright/screenshots/**/*.yml
           push_options: --force-with-lease
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: "Changes detected"
-        if: steps.auto-commit.outputs.changes_detected == 'true'
+        if: ${{ env.REGEN == 'true' && steps.auto-commit.outputs.changes_detected == 'true' }}
         uses: phulsechinmay/rewritable-pr-comment@a1b041997fa84ec3e524ee9dee5ba0b5f4e73604 # v0.3.0
         with:
           message: |
@@ -92,11 +106,10 @@ jobs:
             [Review differences](https://github.com/opf/primer_view_components/pull/${{ github.event.number }}/files?file-filters%5B%5D=.png&file-filters%5B%5D=.yml&show-viewed-files=false)
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COMMENT_IDENTIFIER: "visual-comparison-diff"
+      # Uploaded on assert failures too, so authors can inspect the diff report.
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: ${{ !cancelled() }}
         with:
           name: playwright-report
           path: .playwright/report/
           retention-days: 30
-      - name: Failure
-        if: ${{ steps.auto-commit.outputs.changes_detected == 'true' || steps.playwright-run.outcome == 'failure' }}
-        run: exit 1

playwright.config.ts

@@ -18,7 +18,10 @@ const config: PlaywrightTestConfig = {
   /* Run tests in files in parallel */
   fullyParallel: true,
   workers: process.env.CI ? 4 : undefined,
-  updateSnapshots: 'all',
+  // Assert against the committed baseline by default (Playwright's 'missing').
+  // Snapshots are regenerated only on intent via `--update-snapshots` (the
+  // `regen-snapshots` PR label in test-visual.yml), never unconditionally — an
+  // unconditional 'all' rewrites every file every run and commits byte-churn.
   use: {
     baseURL: 'http://127.0.0.1:4000',
     browserName: 'chromium',
@@ -30,7 +33,12 @@ const config: PlaywrightTestConfig = {
       animations: 'disabled',
     },
     toMatchSnapshot: {
+      // `threshold` sets per-pixel colour sensitivity; `maxDiffPixelRatio` sets
+      // how many pixels may differ before failing. Without a max-diff budget a
+      // single anti-aliased pixel of non-deterministic Chromium rasterisation
+      // fails the run. Tune if genuine small changes slip through.
       threshold: 0.1,
+      maxDiffPixelRatio: 0.01,
     },
   },
   /* Retry on CI only */

script/run-playwright

@@ -1,4 +1,5 @@
 #! /bin/bash
 
 ./script/stop-existing-processes
-exec npx playwright test --workers 6
+# Forward extra args (e.g. --update-snapshots for the regen path in CI).
+exec npx playwright test --workers 6 "$@"