feat(metrics): characterize-metric-anomaly endpoint and MCP tool #164648
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Storybook | |
| on: | |
| pull_request: | |
| # Draft PRs run a narrowed visual-regression matrix (affected stories only); | |
| # ready PRs run the full chromium matrix (the merge gate). ready_for_review | |
| # re-triggers the full run on the current head when a PR leaves draft. To | |
| # force the full matrix on a draft, add the `run-ci-frontend` label — | |
| # labeled/unlabeled re-trigger the run so it starts without needing a push. | |
| types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] | |
| merge_group: | |
| push: | |
| branches: | |
| - master | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Job to decide if we should run storybook ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| name: Determine need to run storybook checks | |
| if: github.event_name != 'merge_group' | |
| # Set job outputs to values from filter step | |
| outputs: | |
| # With dorny's default predicate-quantifier (some), '!path' matches | |
| # every file NOT at that path — effectively matching ALL changes | |
| # including backend files. Instead we use two positive filters and | |
| # compare counts to exclude generated-only PRs. | |
| # | |
| # Additionally skip when a bot commits only frontend/snapshots.yml on | |
| # a PR — that file is updated automatically after human review and CI | |
| # approval, so re-running the full storybook suite is pure waste. | |
| frontend: >- | |
| ${{ | |
| github.event_name == 'push' | |
| || (steps.filter.outputs.frontend == 'true' | |
| && fromJSON(steps.filter.outputs.frontend_count || '0') | |
| > fromJSON(steps.filter.outputs.frontend_generated_count || '0') | |
| && !(github.event_name == 'pull_request' | |
| && (contains(github.actor, '[bot]') || github.actor == 'posthog-bot') | |
| && fromJSON(steps.filter.outputs.snapshots_count || '0') | |
| == fromJSON(steps.filter.outputs.frontend_count || '0'))) | |
| }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| id: app-token | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }} | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| if: github.event_name != 'push' # Run all tests on master push | |
| with: | |
| token: ${{ steps.app-token.outputs.token || github.token }} | |
| filters: | | |
| frontend: | |
| - 'frontend/**' | |
| - 'products/**/*.{ts,tsx}' | |
| - 'products/**/frontend/**' | |
| - 'common/{esbuilder,mosaic,storybook,tailwind}/**' | |
| - 'ee/frontend/**' | |
| - 'services/mcp/src/ui-apps/**' | |
| - '.storybook/**' | |
| - 'package.json' | |
| - '.github/workflows/ci-storybook.yml' | |
| - 'playwright.config.ts' | |
| frontend_generated: | |
| - 'frontend/src/generated/**' | |
| - 'products/**/frontend/generated/**' | |
| snapshots: | |
| - 'frontend/snapshots.yml' | |
| # Build the visual-regression matrix. | |
| # On PRs, only run chromium shards — no story currently opts into webkit snapshots | |
| # (see common/storybook/.storybook/test-runner.ts), so webkit shards consume CI | |
| # minutes without producing snapshot diffs. WebKit render-error coverage is | |
| # preserved on master pushes. | |
| - name: Build matrix | |
| id: matrix | |
| run: | | |
| # Shard counts — change here to resize the matrix. | |
| CHROMIUM_SHARDS=16 | |
| WEBKIT_SHARDS=4 | |
| shards() { | |
| jq -cn --arg browser "$1" --argjson count "$2" \ | |
| '[range(1; $count + 1) | {browser: $browser, shard_count: $count, shard: .}]' | |
| } | |
| CHROMIUM=$(shards chromium "$CHROMIUM_SHARDS") | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| MATRIX=$(jq -cn --argjson c "$CHROMIUM" '{include: $c}') | |
| else | |
| WEBKIT=$(shards webkit "$WEBKIT_SHARDS") | |
| MATRIX=$(jq -cn --argjson c "$CHROMIUM" --argjson w "$WEBKIT" '{include: ($c + $w)}') | |
| fi | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| build-storybook: | |
| name: Build Storybook (depot-ubuntu-latest) | |
| runs-on: depot-ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: changes | |
| if: needs.changes.outputs.frontend == 'true' | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| cache: pnpm | |
| cache-dependency-path: | | |
| pnpm-lock.yaml | |
| .github/workflows/ci-storybook.yml | |
| token: ${{ github.token }} | |
| - name: Install dependencies | |
| run: pnpm --filter=@posthog/storybook... install --frozen-lockfile | |
| - name: Restore webpack build cache | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: common/storybook/node_modules/.cache/ | |
| key: ${{ runner.os }}-webpack-storybook-${{ hashFiles('pnpm-lock.yaml') }} | |
| restore-keys: ${{ runner.os }}-webpack-storybook- | |
| - name: Build Storybook | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=32768 | |
| run: | | |
| bin/turbo --filter=@posthog/storybook prepare | |
| pnpm --filter=@posthog/storybook build --test | |
| - name: Save webpack build cache | |
| if: github.ref == 'refs/heads/master' | |
| uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: common/storybook/node_modules/.cache/ | |
| key: ${{ runner.os }}-webpack-storybook-${{ hashFiles('pnpm-lock.yaml') }} | |
| - name: Upload Storybook build artifact | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: storybook-build | |
| path: common/storybook/dist | |
| retention-days: 1 | |
| # Consumed by the select-stories job below. Uploaded separately so | |
| # the visual-regression matrix doesn't pay to download it on every | |
| # shard. | |
| - name: Upload module-graph artifact | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: storybook-module-graph | |
| path: common/storybook/dist/module-graph.json | |
| retention-days: 1 | |
| if-no-files-found: warn | |
| vr-setup: | |
| name: Create Visual Review run | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: [changes, build-storybook, select-stories] | |
| # always() so vr-setup still runs when select-stories is skipped (ready PRs, | |
| # master push, merge_group). Without it, GitHub treats a skipped need as a | |
| # skip-cascade and vr-setup wouldn't run when we still want it to. | |
| if: | | |
| always() | |
| && needs.changes.outputs.frontend == 'true' | |
| && needs.build-storybook.result == 'success' | |
| && needs.select-stories.outputs.should_run != 'false' | |
| && (github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push') | |
| outputs: | |
| run_id: ${{ steps.create.outputs.run_id }} | |
| steps: | |
| # Head checkout supplies the PR's snapshot baseline; the VR CLI itself is | |
| # built from the base branch below. | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| sparse-checkout: | | |
| .nvmrc | |
| frontend/snapshots.yml | |
| sparse-checkout-cone-mode: false | |
| # Build the VR CLI from the base branch, not the PR head. A workflow edit | |
| # reaches every open PR immediately (it runs PR-merged-with-base), but a CLI | |
| # change only lands on a branch once it rebases — so building from head can | |
| # feed an old binary flags this workflow added (e.g. --partial), failing the | |
| # run. Pinning to base.ref keeps the CLI in lockstep with the workflow. The | |
| # tsconfig rootDir is the parent dir, so the generated types must come along. | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref || github.sha }} | |
| path: vr-cli | |
| sparse-checkout: | | |
| products/visual_review/cli | |
| products/visual_review/frontend/generated/api.schemas.ts | |
| sparse-checkout-cone-mode: false | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| token: ${{ github.token }} | |
| - name: Install VR CLI | |
| run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link | |
| - name: Create VR run | |
| id: create | |
| env: | |
| VR_TOKEN: ${{ secrets.VR_API_TOKEN }} | |
| VR_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }} | |
| VR_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} | |
| VR_PR: ${{ github.event.pull_request.number }} | |
| # PRs are gating ("review"); master pushes are tracking-only ("observe") since | |
| # there's no PR to approve and we don't want master runs to block or prompt for approval. | |
| VR_PURPOSE: ${{ github.event_name == 'push' && 'observe' || 'review' }} | |
| # Selective storybook runs only render a subset of stories. Without --partial, | |
| # VR would flag every untouched baseline identifier as removed. | |
| VR_PARTIAL: ${{ needs.select-stories.outputs.mode == 'selective' && '--partial' || '' }} | |
| # Recorded on the run so the VR UI can re-trigger this job via the GitHub API. | |
| JOB_CHECK_RUN_ID: ${{ job.check_run_id }} | |
| run: | | |
| # shellcheck disable=SC2086 | |
| RUN_ID=$(vr run create \ | |
| --type storybook \ | |
| --baseline frontend/snapshots.yml \ | |
| --branch "$VR_BRANCH" \ | |
| --commit "$VR_COMMIT" \ | |
| --pr "$VR_PR" \ | |
| --purpose "$VR_PURPOSE" \ | |
| --token "$VR_TOKEN" \ | |
| $VR_PARTIAL) | |
| echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT | |
| visual-regression: | |
| name: Visual regression tests - ${{ matrix.browser }} (${{ matrix.shard }}/${{ matrix.shard_count }}) | |
| runs-on: ${{ matrix.browser == 'webkit' && 'depot-ubuntu-latest' || 'ubuntu-latest' }} | |
| needs: [changes, build-storybook, vr-setup, select-stories] | |
| if: >- | |
| always() | |
| && needs.changes.outputs.frontend == 'true' | |
| && needs.build-storybook.result == 'success' | |
| && needs.select-stories.outputs.should_run != 'false' | |
| timeout-minutes: 60 | |
| container: | |
| image: ghcr.io/posthog/playwright:v1.60.0 | |
| strategy: | |
| fail-fast: false | |
| # Shard counts and worker config (maxWorkers in common/storybook/package.json) | |
| # are tuned for runner CPU — consult #team-devex before changing. | |
| # On draft PRs, select-stories may narrow the matrix to a single chromium | |
| # shard running just the affected story files. On ready PRs and master push | |
| # it's skipped, so we fall back to changes.outputs.matrix (ready PRs: | |
| # full chromium; master push: chromium + webkit). | |
| # Storybook CI is skipped entirely on merge_group (changes.if). | |
| matrix: ${{ fromJson(needs.select-stories.outputs.matrix || needs.changes.outputs.matrix) }} | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=16384 | |
| OPT_OUT_CAPTURE: 1 | |
| JEST_JUNIT_SUITE_NAME: '{filepath}' | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| token: ${{ github.token }} | |
| - name: Get pnpm store path | |
| id: pnpm-store | |
| run: echo "path=$(pnpm store path --silent)" >> $GITHUB_OUTPUT | |
| - name: Restore pnpm cache | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: ${{ steps.pnpm-store.outputs.path }} | |
| key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} | |
| restore-keys: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm- | |
| - name: Install package.json dependencies with pnpm | |
| run: pnpm --filter=@posthog/storybook... install --frozen-lockfile | |
| - name: Save pnpm cache | |
| if: github.ref == 'refs/heads/master' | |
| uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: ${{ steps.pnpm-store.outputs.path }} | |
| key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} | |
| - name: Download Storybook build artifact | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: storybook-build | |
| path: common/storybook/dist | |
| - name: Serve Storybook in the background | |
| # http-server and wait-on are devDependencies of @posthog/storybook. | |
| # Run via `pnpm --filter` so pnpm resolves them from common/storybook/node_modules/.bin | |
| # (no global install) and cd's into the package — `dist` is relative to common/storybook. | |
| run: | | |
| retries=5 | |
| max_timeout=30 | |
| pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent & | |
| server_pid=$! | |
| echo "Started http-server with PID: $server_pid" | |
| # Give the server a moment to start | |
| sleep 2 | |
| while [ $retries -gt 0 ]; do | |
| echo "Checking if Storybook is available (retries left: $retries, timeout: ${max_timeout}s)..." | |
| if pnpm --filter=@posthog/storybook exec wait-on http://127.0.0.1:6006 --timeout $max_timeout; then | |
| echo "✅ Storybook is available at http://127.0.0.1:6006" | |
| break | |
| fi | |
| retries=$((retries-1)) | |
| if [ $retries -gt 0 ]; then | |
| echo "⚠️ Failed to connect to Storybook, retrying... ($retries retries left)" | |
| # Check if server is still running | |
| if ! kill -0 $server_pid 2>/dev/null; then | |
| echo "❌ http-server process is no longer running, restarting it..." | |
| pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent & | |
| server_pid=$! | |
| echo "Restarted http-server with PID: $server_pid" | |
| sleep 2 | |
| fi | |
| fi | |
| done | |
| if [ $retries -eq 0 ]; then | |
| echo "❌ Failed to serve Storybook after all retries" | |
| # Try to get some diagnostic information | |
| echo "Checking port 6006 status:" | |
| netstat -tuln | grep 6006 || echo "Port 6006 is not in use" | |
| echo "Checking http-server process:" | |
| ps aux | grep http-server || echo "No http-server process found" | |
| echo "Checking Storybook dist directory:" | |
| ls -la common/storybook/dist || echo "Storybook dist directory not found" | |
| exit 1 | |
| fi | |
| # Wipe committed PNGs so each shard only has freshly captured screenshots. | |
| # Prevents stale hashes from winning the race in VR's get_or_create. | |
| - name: Clean snapshot directory | |
| run: find frontend/__snapshots__ -name '*.png' -delete 2>/dev/null || true | |
| # Capture-only: VR is the gate for visual changes, jest just captures screenshots | |
| - name: Run @storybook/test-runner | |
| id: test-runner | |
| shell: bash | |
| env: | |
| HOME: /root | |
| STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}' | |
| MODE: ${{ needs.select-stories.outputs.mode }} | |
| AFFECTED_FILES: ${{ needs.select-stories.outputs.affected_files }} | |
| run: | | |
| if [ "$MODE" = "selective" ]; then | |
| # test-storybook is jest-based; positional path args filter the run to | |
| # only those story files. Single shard, so --shard 1/1 is a no-op. | |
| # shellcheck disable=SC2086 | |
| pnpm --filter=@posthog/storybook test:visual:ci:update --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/${{ matrix.shard_count }} -- $AFFECTED_FILES | |
| else | |
| pnpm --filter=@posthog/storybook test:visual:ci:update --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/${{ matrix.shard_count }} | |
| fi | |
| # Upload snapshots to Visual Review (runs in parallel across shards). | |
| # Build the CLI from the base branch (see vr-setup for the rationale). | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| if: always() && needs.vr-setup.outputs.run_id != '' | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref || github.sha }} | |
| path: vr-cli | |
| sparse-checkout: | | |
| products/visual_review/cli | |
| products/visual_review/frontend/generated/api.schemas.ts | |
| sparse-checkout-cone-mode: false | |
| - name: Install VR CLI | |
| if: always() && needs.vr-setup.outputs.run_id != '' | |
| run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link | |
| - name: Upload snapshots to Visual Review | |
| if: always() && needs.vr-setup.outputs.run_id != '' | |
| env: | |
| VR_TOKEN: ${{ secrets.VR_API_TOKEN }} | |
| run: | | |
| vr run upload \ | |
| --run-id "${{ needs.vr-setup.outputs.run_id }}" \ | |
| --dir frontend/__snapshots__/ \ | |
| --baseline frontend/snapshots.yml \ | |
| --token "$VR_TOKEN" | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: junit-results-storybook-${{ matrix.browser }}-${{ matrix.shard }} | |
| path: common/storybook/junit.xml | |
| if-no-files-found: ignore | |
| # Verify changed story files are stable by re-running them multiple times. | |
| # Catches flaky snapshots before they land on master. | |
| flake-verification: | |
| name: Storybook flake verification | |
| runs-on: ubuntu-latest | |
| needs: [changes, build-storybook] | |
| if: needs.changes.outputs.frontend == 'true' && github.event_name == 'pull_request' | |
| timeout-minutes: 30 | |
| container: | |
| image: ghcr.io/posthog/playwright:v1.60.0 | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=16384 | |
| OPT_OUT_CAPTURE: 1 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| token: ${{ github.token }} | |
| - name: Get pnpm store path | |
| id: pnpm-store | |
| run: echo "path=$(pnpm store path --silent)" >> $GITHUB_OUTPUT | |
| - name: Restore pnpm cache | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: ${{ steps.pnpm-store.outputs.path }} | |
| key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} | |
| restore-keys: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm- | |
| - name: Install dependencies | |
| run: pnpm --filter=@posthog/storybook... install --frozen-lockfile | |
| - name: Download Storybook build artifact | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: storybook-build | |
| path: common/storybook/dist | |
| - name: Serve Storybook | |
| # Resolve http-server/wait-on from common/storybook's devDependencies (no global install). | |
| run: | | |
| pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent & | |
| sleep 2 | |
| pnpm --filter=@posthog/storybook exec wait-on http://127.0.0.1:6006 --timeout 30 | |
| - name: Verify changed stories are stable | |
| id: flake-verify | |
| env: | |
| HOME: /root | |
| run: | | |
| git config --global --add safe.directory '*' | |
| BASE_SHA="${{ github.event.pull_request.base.sha }}" | |
| git fetch --no-tags --prune --depth=50 origin "$BASE_SHA" | |
| .github/scripts/verify-storybook-new-stories.sh "$BASE_SHA" 3 | |
| - name: Upload flake verification screenshots | |
| if: failure() && steps.flake-verify.outcome == 'failure' | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: flake-verification-diffs | |
| path: | | |
| frontend/__snapshots__/__diff_output__/ | |
| frontend/__snapshots__/__failures__/ | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| # Collate matrix + VR completion status for the required check | |
| visual_regression_tests: | |
| needs: [visual-regression, vr-setup, vr-complete] | |
| name: Visual regression tests pass | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Check matrix outcome | |
| run: | | |
| if [[ "${{ needs.visual-regression.result }}" != "success" && "${{ needs.visual-regression.result }}" != "skipped" ]]; then | |
| echo "One or more jobs in the visual-regression test matrix failed." | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.vr-setup.result }}" == "failure" ]]; then | |
| echo "Visual Review setup failed — VR CLI or API error." | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.vr-complete.result }}" != "success" && "${{ needs.vr-complete.result }}" != "skipped" ]]; then | |
| echo "Visual Review did not complete successfully (result: ${{ needs.vr-complete.result }})." | |
| exit 1 | |
| fi | |
| echo "All jobs passed or were skipped successfully." | |
| vr-complete: | |
| name: Complete Visual Review run | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: [visual-regression, vr-setup] | |
| # always() so the skipped select-stories job (transitively in the needs | |
| # chain on ready PRs) doesn't skip-propagate past the always() jobs above | |
| # and leave the VR run uncompleted with its commit status stuck pending. | |
| # Legit skips still work: no-VR cases leave run_id empty, and a failed or | |
| # cancelled matrix fails the result == 'success' check. | |
| if: | | |
| always() | |
| && needs.vr-setup.outputs.run_id != '' | |
| && needs.visual-regression.result == 'success' | |
| steps: | |
| # Head checkout supplies the PR's snapshot baseline; the VR CLI itself is | |
| # built from the base branch below. | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} | |
| sparse-checkout: | | |
| .nvmrc | |
| frontend/snapshots.yml | |
| sparse-checkout-cone-mode: false | |
| # Build the VR CLI from the base branch, not the PR head (see vr-setup | |
| # for the rationale). | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| ref: ${{ github.event.pull_request.base.ref || github.sha }} | |
| path: vr-cli | |
| sparse-checkout: | | |
| products/visual_review/cli | |
| products/visual_review/frontend/generated/api.schemas.ts | |
| sparse-checkout-cone-mode: false | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| token: ${{ github.token }} | |
| - name: Install VR CLI | |
| run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link | |
| - name: Complete Visual Review run | |
| # On master pushes the run was created with --purpose observe (tracking-only), | |
| # but `vr run complete` still exits 1 whenever visual changes are detected. | |
| # That makes the Storybook check fail on master even though observe runs are | |
| # not gating by design. Treat exit 1 as informational on push events; PRs | |
| # still gate as before. | |
| continue-on-error: ${{ github.event_name == 'push' }} | |
| env: | |
| VR_TOKEN: ${{ secrets.VR_API_TOKEN }} | |
| run: | | |
| vr run complete \ | |
| --run-id "${{ needs.vr-setup.outputs.run_id }}" \ | |
| --baseline frontend/snapshots.yml \ | |
| --token "$VR_TOKEN" | |
| calculate-running-time: | |
| name: Calculate running time | |
| needs: [visual-regression, visual_regression_tests, changes] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks or Dependabot (no secrets access) | |
| always() && github.actor != 'dependabot[bot]' && | |
| needs.changes.outputs.frontend == 'true' && ( | |
| (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') || | |
| (github.event_name != 'pull_request' && github.repository == 'PostHog/posthog')) | |
| steps: | |
| - name: Get telemetry app token | |
| id: telemetry-app-token | |
| if: github.run_attempt == '1' | |
| uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| with: | |
| client-id: ${{ secrets.GH_APP_TELEMETRY_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_TELEMETRY_PRIVATE_KEY }} | |
| - name: Capture running time to PostHog | |
| if: github.run_attempt == '1' | |
| continue-on-error: true | |
| uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0 | |
| with: | |
| posthog-token: ${{ secrets.POSTHOG_API_TOKEN }} | |
| event: 'posthog-ci-running-time' | |
| capture-run-duration: true | |
| capture-job-durations: true | |
| github-token: ${{ steps.telemetry-app-token.outputs.token }} | |
| status-job: 'Visual regression tests pass' | |
| - name: Capture running time to DevEx PostHog | |
| if: github.run_attempt == '1' | |
| continue-on-error: true | |
| uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0 | |
| with: | |
| posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }} | |
| event: 'posthog-ci-running-time' | |
| capture-run-duration: true | |
| capture-job-durations: true | |
| github-token: ${{ steps.telemetry-app-token.outputs.token }} | |
| status-job: 'Visual regression tests pass' | |
| # Shadow measurement: reports which stories could have rendered differently | |
| # Pick the visual-regression matrix on draft PRs based on which stories the | |
| # webpack module graph (emitted by ModuleGraphPlugin) says are affected | |
| # by the diff. On ready PRs and master push this job is skipped and | |
| # visual-regression falls back to needs.changes.outputs.matrix (the full | |
| # suite). The ready-for-review full run is the merge gate — storybook | |
| # already skips merge_group entirely (changes.if). The run-ci-frontend | |
| # label forces the full matrix on a draft. When selection can't be trusted | |
| # on a draft, visual-regression is skipped and deferred to the ready run. | |
| select-stories: | |
| name: Story selection | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: [changes, build-storybook] | |
| if: | | |
| always() && needs.changes.outputs.frontend == 'true' | |
| && github.event_name == 'pull_request' | |
| && github.event.pull_request.draft == true | |
| && !contains(github.event.pull_request.labels.*.name, 'run-ci-frontend') | |
| outputs: | |
| mode: ${{ steps.classify.outputs.mode }} | |
| matrix: ${{ steps.classify.outputs.matrix }} | |
| should_run: ${{ steps.classify.outputs.should_run }} | |
| affected_files: ${{ steps.classify.outputs.affected_files }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| filter: blob:none | |
| fetch-depth: 500 | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| token: ${{ github.token }} | |
| - name: Download module-graph artifact | |
| id: download | |
| continue-on-error: true | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: storybook-module-graph | |
| path: /tmp/storybook-graph | |
| - name: Compute affected stories | |
| id: selection | |
| continue-on-error: true | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| run: | | |
| # Fetch the *current* tip of the base branch, not pull_request.base.sha — | |
| # base.sha is captured at webhook time and goes stale if the branch later | |
| # merges a newer master, which makes `base.sha...HEAD` balloon to include | |
| # every merged-in master change. | |
| git fetch --no-tags origin "$BASE_REF" | |
| echo "### Changed files" >&2 | |
| git diff --name-only "origin/$BASE_REF"...HEAD >&2 | |
| RESULT=$(bin/find-affected-stories \ | |
| --graph /tmp/storybook-graph/module-graph.json \ | |
| --base-ref "origin/$BASE_REF") | |
| echo "$RESULT" > /tmp/selection-result.json | |
| # Stash compact form in step output (single line) for the summary step. | |
| COMPACT=$(node -e 'const d = JSON.parse(require("fs").readFileSync("/tmp/selection-result.json", "utf8")); delete d.affected_stories; delete d.unresolved_changed_files; process.stdout.write(JSON.stringify(d))') | |
| echo "result=$COMPACT" >> "$GITHUB_OUTPUT" | |
| - name: Build narrowed matrix | |
| id: classify | |
| env: | |
| SELECT_OUTCOME: ${{ steps.selection.outcome }} | |
| run: | | |
| set -euo pipefail | |
| # Untrusted selection on a draft skips visual-regression entirely | |
| # (should_run=false gates vr-setup and the test matrix off); the | |
| # ready-for-review full run is the gate. A skipped draft can't | |
| # trade green-by-default for green-by-skipping: drafts can't | |
| # merge, and ready re-runs the full suite. The run-ci-frontend | |
| # label forces the full matrix on a draft. | |
| fall_back_to_skip() { | |
| echo "mode=skip" >> "$GITHUB_OUTPUT" | |
| echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT" | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| echo "affected_files=" >> "$GITHUB_OUTPUT" | |
| } | |
| if [[ "$SELECT_OUTCOME" != "success" ]] || [[ ! -s /tmp/selection-result.json ]]; then | |
| echo "::warning::story selector did not produce output; draft skips visual-regression (full run happens on ready for review)" | |
| fall_back_to_skip | |
| exit 0 | |
| fi | |
| MODE=$(jq -r '.mode' /tmp/selection-result.json) | |
| if [[ "$MODE" != "selective" ]]; then | |
| echo "Selector declined to narrow (mode=$MODE); draft skips visual-regression (full run happens on ready for review)" | |
| fall_back_to_skip | |
| exit 0 | |
| fi | |
| COUNT=$(jq -r '.affected_story_count' /tmp/selection-result.json) | |
| if [[ "$COUNT" -eq 0 ]]; then | |
| # Selective mode found zero affected stories — skip visual-regression entirely. | |
| echo "mode=selective" >> "$GITHUB_OUTPUT" | |
| echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT" | |
| echo "should_run=false" >> "$GITHUB_OUTPUT" | |
| echo "affected_files=" >> "$GITHUB_OUTPUT" | |
| echo "Selective mode: 0 affected stories, skipping visual-regression" | |
| exit 0 | |
| fi | |
| # Selective with affected stories: collapse to one chromium shard. | |
| AFFECTED_FILES=$(jq -r '.affected_stories | join(" ")' /tmp/selection-result.json) | |
| echo "mode=selective" >> "$GITHUB_OUTPUT" | |
| echo 'matrix={"include":[{"browser":"chromium","shard":1,"shard_count":1}]}' >> "$GITHUB_OUTPUT" | |
| echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| echo "affected_files=$AFFECTED_FILES" >> "$GITHUB_OUTPUT" | |
| echo "Selective mode: $COUNT affected story files" | |
| - name: Write summary | |
| if: steps.selection.outcome == 'success' | |
| env: | |
| RESULT: ${{ steps.selection.outputs.result }} | |
| CURRENT_SHARDS: 16 | |
| SHARD_OVERHEAD_MINUTES: 3 | |
| run: | | |
| MODE=$(echo "$RESULT" | jq -r '.mode') | |
| TOTAL=$(echo "$RESULT" | jq -r '.total_story_count') | |
| if [ "$MODE" = "selective" ]; then | |
| COUNT=$(echo "$RESULT" | jq -r '.affected_story_count') | |
| AFFECTED_DUR=$(echo "$RESULT" | jq -r '.affected_duration_seconds') | |
| TOTAL_DUR=$(echo "$RESULT" | jq -r '.total_duration_seconds') | |
| # Selected mode collapses to a single chromium shard (see classify step). | |
| SHARDS=1 | |
| SAVED_STORIES=$((TOTAL - COUNT)) | |
| SAVED_SHARDS=$((CURRENT_SHARDS - SHARDS)) | |
| FULL_MINUTES=$(( (CURRENT_SHARDS * SHARD_OVERHEAD_MINUTES) + (TOTAL_DUR / 60) )) | |
| SELECTIVE_MINUTES=$(( (SHARDS * SHARD_OVERHEAD_MINUTES) + (AFFECTED_DUR / 60) )) | |
| SAVED_MINUTES=$((FULL_MINUTES - SELECTIVE_MINUTES)) | |
| cat >> "$GITHUB_STEP_SUMMARY" << EOF | |
| ## Story selection | |
| | Metric | Full run | Selective | Saved | | |
| |---|---|---|---| | |
| | Story files | $TOTAL | $COUNT | **$SAVED_STORIES** | | |
| | Shards | $CURRENT_SHARDS | $SHARDS | **$SAVED_SHARDS** | | |
| | Est. CI minutes | ~${FULL_MINUTES} | ~${SELECTIVE_MINUTES} | **~${SAVED_MINUTES}** | | |
| | Est. render time | ${TOTAL_DUR}s | ${AFFECTED_DUR}s | **$((TOTAL_DUR - AFFECTED_DUR))s** | | |
| > Predictor: webpack module graph from build-storybook. Master push runs the full suite. | |
| EOF | |
| echo "SELECTION_METRICS: $(echo "$RESULT" | jq -c '{mode, affected: .affected_story_count, total: .total_story_count, current_shards: '$CURRENT_SHARDS', affected_dur: .affected_duration_seconds, total_dur: .total_duration_seconds}')" | |
| else | |
| REASON=$(echo "$RESULT" | jq -r '.reason // "unknown"') | |
| cat >> "$GITHUB_STEP_SUMMARY" << EOF | |
| ## Story selection | |
| **Full run required:** $REASON | |
| Total stories: $TOTAL | |
| EOF | |
| echo "SELECTION_METRICS: $(echo "$RESULT" | jq -c '{mode, reason, total: .total_story_count}')" | |
| fi |