Skip to content
Storybook

feat(metrics): expose query-metrics and metric-names-list MCP tools #164431

Sign in to view logs

feat(metrics): expose query-metrics and metric-names-list MCP tools

feat(metrics): expose query-metrics and metric-names-list MCP tools #164431

Workflow file for this run

.github/workflows/ci-storybook.yml at f020aa1
name: Storybook
on:
pull_request:
# Draft PRs run a narrowed visual-regression matrix (affected stories only);
# ready PRs run the full chromium matrix (the merge gate). ready_for_review
# re-triggers the full run on the current head when a PR leaves draft. To
# force the full matrix on a draft, add the `run-ci-frontend` label —
# labeled/unlabeled re-trigger the run so it starts without needing a push.
types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
merge_group:
push:
branches:
- master
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
jobs:
# Job to decide if we should run storybook ci
# See https://github.com/dorny/paths-filter#conditional-execution for more details
changes:
runs-on: ubuntu-latest
timeout-minutes: 5
name: Determine need to run storybook checks
if: github.event_name != 'merge_group'
# Set job outputs to values from filter step
outputs:
# With dorny's default predicate-quantifier (some), '!path' matches
# every file NOT at that path — effectively matching ALL changes
# including backend files. Instead we use two positive filters and
# compare counts to exclude generated-only PRs.
#
# Additionally skip when a bot commits only frontend/snapshots.yml on
# a PR — that file is updated automatically after human review and CI
# approval, so re-running the full storybook suite is pure waste.
frontend: >-
${{
github.event_name == 'push'
|| (steps.filter.outputs.frontend == 'true'
&& fromJSON(steps.filter.outputs.frontend_count || '0')
> fromJSON(steps.filter.outputs.frontend_generated_count || '0')
&& !(github.event_name == 'pull_request'
&& (contains(github.actor, '[bot]') || github.actor == 'posthog-bot')
&& fromJSON(steps.filter.outputs.snapshots_count || '0')
== fromJSON(steps.filter.outputs.frontend_count || '0')))
}}
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
id: app-token
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
with:
client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }}
private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }}
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
id: filter
if: github.event_name != 'push' # Run all tests on master push
with:
token: ${{ steps.app-token.outputs.token || github.token }}
filters: |
frontend:
- 'frontend/**'
- 'products/**/*.{ts,tsx}'
- 'products/**/frontend/**'
- 'common/{esbuilder,mosaic,storybook,tailwind}/**'
- 'ee/frontend/**'
- 'services/mcp/src/ui-apps/**'
- '.storybook/**'
- 'package.json'
- '.github/workflows/ci-storybook.yml'
- 'playwright.config.ts'
frontend_generated:
- 'frontend/src/generated/**'
- 'products/**/frontend/generated/**'
snapshots:
- 'frontend/snapshots.yml'
# Build the visual-regression matrix.
# On PRs, only run chromium shards — no story currently opts into webkit snapshots
# (see common/storybook/.storybook/test-runner.ts), so webkit shards consume CI
# minutes without producing snapshot diffs. WebKit render-error coverage is
# preserved on master pushes.
- name: Build matrix
id: matrix
run: |
# Shard counts — change here to resize the matrix.
CHROMIUM_SHARDS=16
WEBKIT_SHARDS=4
shards() {
jq -cn --arg browser "$1" --argjson count "$2" \
'[range(1; $count + 1) | {browser: $browser, shard_count: $count, shard: .}]'
}
CHROMIUM=$(shards chromium "$CHROMIUM_SHARDS")
if [ "${{ github.event_name }}" = "pull_request" ]; then
MATRIX=$(jq -cn --argjson c "$CHROMIUM" '{include: $c}')
else
WEBKIT=$(shards webkit "$WEBKIT_SHARDS")
MATRIX=$(jq -cn --argjson c "$CHROMIUM" --argjson w "$WEBKIT" '{include: ($c + $w)}')
fi
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
build-storybook:
name: Build Storybook (depot-ubuntu-latest)
runs-on: depot-ubuntu-latest
timeout-minutes: 15
needs: changes
if: needs.changes.outputs.frontend == 'true'
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
- name: Install pnpm
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
- name: Set up Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
cache: pnpm
cache-dependency-path: |
pnpm-lock.yaml
.github/workflows/ci-storybook.yml
token: ${{ github.token }}
- name: Install dependencies
run: pnpm --filter=@posthog/storybook... install --frozen-lockfile
- name: Restore webpack build cache
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: common/storybook/node_modules/.cache/
key: ${{ runner.os }}-webpack-storybook-${{ hashFiles('pnpm-lock.yaml') }}
restore-keys: ${{ runner.os }}-webpack-storybook-
- name: Build Storybook
env:
NODE_OPTIONS: --max-old-space-size=32768
run: |
bin/turbo --filter=@posthog/storybook prepare
pnpm --filter=@posthog/storybook build --test
- name: Save webpack build cache
if: github.ref == 'refs/heads/master'
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: common/storybook/node_modules/.cache/
key: ${{ runner.os }}-webpack-storybook-${{ hashFiles('pnpm-lock.yaml') }}
- name: Upload Storybook build artifact
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: storybook-build
path: common/storybook/dist
retention-days: 1
# Consumed by the select-stories job below. Uploaded separately so
# the visual-regression matrix doesn't pay to download it on every
# shard.
- name: Upload module-graph artifact
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: storybook-module-graph
path: common/storybook/dist/module-graph.json
retention-days: 1
if-no-files-found: warn
vr-setup:
name: Create Visual Review run
runs-on: ubuntu-latest
timeout-minutes: 5
needs: [changes, build-storybook, select-stories]
# always() so vr-setup still runs when select-stories is skipped (ready PRs,
# master push, merge_group). Without it, GitHub treats a skipped need as a
# skip-cascade and vr-setup wouldn't run when we still want it to.
if: |
always()
&& needs.changes.outputs.frontend == 'true'
&& needs.build-storybook.result == 'success'
&& needs.select-stories.outputs.should_run != 'false'
&& (github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push')
outputs:
run_id: ${{ steps.create.outputs.run_id }}
steps:
# Head checkout supplies the PR's snapshot baseline; the VR CLI itself is
# built from the base branch below.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
sparse-checkout: |
.nvmrc
frontend/snapshots.yml
sparse-checkout-cone-mode: false
# Build the VR CLI from the base branch, not the PR head. A workflow edit
# reaches every open PR immediately (it runs PR-merged-with-base), but a CLI
# change only lands on a branch once it rebases — so building from head can
# feed an old binary flags this workflow added (e.g. --partial), failing the
# run. Pinning to base.ref keeps the CLI in lockstep with the workflow. The
# tsconfig rootDir is the parent dir, so the generated types must come along.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.base.ref || github.sha }}
path: vr-cli
sparse-checkout: |
products/visual_review/cli
products/visual_review/frontend/generated/api.schemas.ts
sparse-checkout-cone-mode: false
- name: Set up Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
token: ${{ github.token }}
- name: Install VR CLI
run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link
- name: Create VR run
id: create
env:
VR_TOKEN: ${{ secrets.VR_API_TOKEN }}
VR_BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }}
VR_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
VR_PR: ${{ github.event.pull_request.number }}
# PRs are gating ("review"); master pushes are tracking-only ("observe") since
# there's no PR to approve and we don't want master runs to block or prompt for approval.
VR_PURPOSE: ${{ github.event_name == 'push' && 'observe' || 'review' }}
# Selective storybook runs only render a subset of stories. Without --partial,
# VR would flag every untouched baseline identifier as removed.
VR_PARTIAL: ${{ needs.select-stories.outputs.mode == 'selective' && '--partial' || '' }}
# Recorded on the run so the VR UI can re-trigger this job via the GitHub API.
JOB_CHECK_RUN_ID: ${{ job.check_run_id }}
run: |
# shellcheck disable=SC2086
RUN_ID=$(vr run create \
--type storybook \
--baseline frontend/snapshots.yml \
--branch "$VR_BRANCH" \
--commit "$VR_COMMIT" \
--pr "$VR_PR" \
--purpose "$VR_PURPOSE" \
--token "$VR_TOKEN" \
$VR_PARTIAL)
echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
visual-regression:
name: Visual regression tests - ${{ matrix.browser }} (${{ matrix.shard }}/${{ matrix.shard_count }})
runs-on: ${{ matrix.browser == 'webkit' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
needs: [changes, build-storybook, vr-setup, select-stories]
if: >-
always()
&& needs.changes.outputs.frontend == 'true'
&& needs.build-storybook.result == 'success'
&& needs.select-stories.outputs.should_run != 'false'
timeout-minutes: 60
container:
image: ghcr.io/posthog/playwright:v1.60.0
strategy:
fail-fast: false
# Shard counts and worker config (maxWorkers in common/storybook/package.json)
# are tuned for runner CPU — consult #team-devex before changing.
# On draft PRs, select-stories may narrow the matrix to a single chromium
# shard running just the affected story files. On ready PRs and master push
# it's skipped, so we fall back to changes.outputs.matrix (ready PRs:
# full chromium; master push: chromium + webkit).
# Storybook CI is skipped entirely on merge_group (changes.if).
matrix: ${{ fromJson(needs.select-stories.outputs.matrix || needs.changes.outputs.matrix) }}
env:
NODE_OPTIONS: --max-old-space-size=16384
OPT_OUT_CAPTURE: 1
JEST_JUNIT_SUITE_NAME: '{filepath}'
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
- name: Install pnpm
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
- name: Set up Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
token: ${{ github.token }}
- name: Get pnpm store path
id: pnpm-store
run: echo "path=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
- name: Restore pnpm cache
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
restore-keys: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-
- name: Install package.json dependencies with pnpm
run: pnpm --filter=@posthog/storybook... install --frozen-lockfile
- name: Save pnpm cache
if: github.ref == 'refs/heads/master'
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
- name: Download Storybook build artifact
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
name: storybook-build
path: common/storybook/dist
- name: Serve Storybook in the background
# http-server and wait-on are devDependencies of @posthog/storybook.
# Run via `pnpm --filter` so pnpm resolves them from common/storybook/node_modules/.bin
# (no global install) and cd's into the package — `dist` is relative to common/storybook.
run: |
retries=5
max_timeout=30
pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent &
server_pid=$!
echo "Started http-server with PID: $server_pid"
# Give the server a moment to start
sleep 2
while [ $retries -gt 0 ]; do
echo "Checking if Storybook is available (retries left: $retries, timeout: ${max_timeout}s)..."
if pnpm --filter=@posthog/storybook exec wait-on http://127.0.0.1:6006 --timeout $max_timeout; then
echo "✅ Storybook is available at http://127.0.0.1:6006"
break
fi
retries=$((retries-1))
if [ $retries -gt 0 ]; then
echo "⚠️ Failed to connect to Storybook, retrying... ($retries retries left)"
# Check if server is still running
if ! kill -0 $server_pid 2>/dev/null; then
echo "❌ http-server process is no longer running, restarting it..."
pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent &
server_pid=$!
echo "Restarted http-server with PID: $server_pid"
sleep 2
fi
fi
done
if [ $retries -eq 0 ]; then
echo "❌ Failed to serve Storybook after all retries"
# Try to get some diagnostic information
echo "Checking port 6006 status:"
netstat -tuln | grep 6006 || echo "Port 6006 is not in use"
echo "Checking http-server process:"
ps aux | grep http-server || echo "No http-server process found"
echo "Checking Storybook dist directory:"
ls -la common/storybook/dist || echo "Storybook dist directory not found"
exit 1
fi
# Wipe committed PNGs so each shard only has freshly captured screenshots.
# Prevents stale hashes from winning the race in VR's get_or_create.
- name: Clean snapshot directory
run: find frontend/__snapshots__ -name '*.png' -delete 2>/dev/null || true
# Capture-only: VR is the gate for visual changes, jest just captures screenshots
- name: Run @storybook/test-runner
id: test-runner
shell: bash
env:
HOME: /root
STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}'
MODE: ${{ needs.select-stories.outputs.mode }}
AFFECTED_FILES: ${{ needs.select-stories.outputs.affected_files }}
run: |
if [ "$MODE" = "selective" ]; then
# test-storybook is jest-based; positional path args filter the run to
# only those story files. Single shard, so --shard 1/1 is a no-op.
# shellcheck disable=SC2086
pnpm --filter=@posthog/storybook test:visual:ci:update --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/${{ matrix.shard_count }} -- $AFFECTED_FILES
else
pnpm --filter=@posthog/storybook test:visual:ci:update --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/${{ matrix.shard_count }}
fi
# Upload snapshots to Visual Review (runs in parallel across shards).
# Build the CLI from the base branch (see vr-setup for the rationale).
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
if: always() && needs.vr-setup.outputs.run_id != ''
with:
ref: ${{ github.event.pull_request.base.ref || github.sha }}
path: vr-cli
sparse-checkout: |
products/visual_review/cli
products/visual_review/frontend/generated/api.schemas.ts
sparse-checkout-cone-mode: false
- name: Install VR CLI
if: always() && needs.vr-setup.outputs.run_id != ''
run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link
- name: Upload snapshots to Visual Review
if: always() && needs.vr-setup.outputs.run_id != ''
env:
VR_TOKEN: ${{ secrets.VR_API_TOKEN }}
run: |
vr run upload \
--run-id "${{ needs.vr-setup.outputs.run_id }}" \
--dir frontend/__snapshots__/ \
--baseline frontend/snapshots.yml \
--token "$VR_TOKEN"
- name: Upload test results
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: junit-results-storybook-${{ matrix.browser }}-${{ matrix.shard }}
path: common/storybook/junit.xml
if-no-files-found: ignore
# Verify changed story files are stable by re-running them multiple times.
# Catches flaky snapshots before they land on master.
flake-verification:
name: Storybook flake verification
runs-on: ubuntu-latest
needs: [changes, build-storybook]
if: needs.changes.outputs.frontend == 'true' && github.event_name == 'pull_request'
timeout-minutes: 30
container:
image: ghcr.io/posthog/playwright:v1.60.0
env:
NODE_OPTIONS: --max-old-space-size=16384
OPT_OUT_CAPTURE: 1
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Install pnpm
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
- name: Set up Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
token: ${{ github.token }}
- name: Get pnpm store path
id: pnpm-store
run: echo "path=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
- name: Restore pnpm cache
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ${{ steps.pnpm-store.outputs.path }}
key: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
restore-keys: node-cache-${{ runner.os }}-${{ runner.arch }}-pnpm-
- name: Install dependencies
run: pnpm --filter=@posthog/storybook... install --frozen-lockfile
- name: Download Storybook build artifact
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
name: storybook-build
path: common/storybook/dist
- name: Serve Storybook
# Resolve http-server/wait-on from common/storybook's devDependencies (no global install).
run: |
pnpm --filter=@posthog/storybook exec http-server dist --port 6006 --silent &
sleep 2
pnpm --filter=@posthog/storybook exec wait-on http://127.0.0.1:6006 --timeout 30
- name: Verify changed stories are stable
id: flake-verify
env:
HOME: /root
run: |
git config --global --add safe.directory '*'
BASE_SHA="${{ github.event.pull_request.base.sha }}"
git fetch --no-tags --prune --depth=50 origin "$BASE_SHA"
.github/scripts/verify-storybook-new-stories.sh "$BASE_SHA" 3
- name: Upload flake verification screenshots
if: failure() && steps.flake-verify.outcome == 'failure'
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: flake-verification-diffs
path: |
frontend/__snapshots__/__diff_output__/
frontend/__snapshots__/__failures__/
if-no-files-found: ignore
retention-days: 5
# Collate matrix + VR completion status for the required check
visual_regression_tests:
needs: [visual-regression, vr-setup, vr-complete]
name: Visual regression tests pass
runs-on: ubuntu-latest
timeout-minutes: 5
if: always()
steps:
- name: Check matrix outcome
run: |
if [[ "${{ needs.visual-regression.result }}" != "success" && "${{ needs.visual-regression.result }}" != "skipped" ]]; then
echo "One or more jobs in the visual-regression test matrix failed."
exit 1
fi
if [[ "${{ needs.vr-setup.result }}" == "failure" ]]; then
echo "Visual Review setup failed — VR CLI or API error."
exit 1
fi
if [[ "${{ needs.vr-complete.result }}" != "success" && "${{ needs.vr-complete.result }}" != "skipped" ]]; then
echo "Visual Review did not complete successfully (result: ${{ needs.vr-complete.result }})."
exit 1
fi
echo "All jobs passed or were skipped successfully."
vr-complete:
name: Complete Visual Review run
runs-on: ubuntu-latest
timeout-minutes: 15
needs: [visual-regression, vr-setup]
# always() so the skipped select-stories job (transitively in the needs
# chain on ready PRs) doesn't skip-propagate past the always() jobs above
# and leave the VR run uncompleted with its commit status stuck pending.
# Legit skips still work: no-VR cases leave run_id empty, and a failed or
# cancelled matrix fails the result == 'success' check.
if: |
always()
&& needs.vr-setup.outputs.run_id != ''
&& needs.visual-regression.result == 'success'
steps:
# Head checkout supplies the PR's snapshot baseline; the VR CLI itself is
# built from the base branch below.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
sparse-checkout: |
.nvmrc
frontend/snapshots.yml
sparse-checkout-cone-mode: false
# Build the VR CLI from the base branch, not the PR head (see vr-setup
# for the rationale).
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ github.event.pull_request.base.ref || github.sha }}
path: vr-cli
sparse-checkout: |
products/visual_review/cli
products/visual_review/frontend/generated/api.schemas.ts
sparse-checkout-cone-mode: false
- name: Set up Node.js
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
token: ${{ github.token }}
- name: Install VR CLI
run: cd vr-cli/products/visual_review/cli && npm ci && npm run build && npm link
- name: Complete Visual Review run
# On master pushes the run was created with --purpose observe (tracking-only),
# but `vr run complete` still exits 1 whenever visual changes are detected.
# That makes the Storybook check fail on master even though observe runs are
# not gating by design. Treat exit 1 as informational on push events; PRs
# still gate as before.
continue-on-error: ${{ github.event_name == 'push' }}
env:
VR_TOKEN: ${{ secrets.VR_API_TOKEN }}
run: |
vr run complete \
--run-id "${{ needs.vr-setup.outputs.run_id }}" \
--baseline frontend/snapshots.yml \
--token "$VR_TOKEN"
calculate-running-time:
name: Calculate running time
needs: [visual-regression, visual_regression_tests, changes]
runs-on: ubuntu-latest
timeout-minutes: 5
if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks or Dependabot (no secrets access)
always() && github.actor != 'dependabot[bot]' &&
needs.changes.outputs.frontend == 'true' && (
(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog') ||
(github.event_name != 'pull_request' && github.repository == 'PostHog/posthog'))
steps:
- name: Get telemetry app token
id: telemetry-app-token
if: github.run_attempt == '1'
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
with:
client-id: ${{ secrets.GH_APP_TELEMETRY_APP_ID }}
private-key: ${{ secrets.GH_APP_TELEMETRY_PRIVATE_KEY }}
- name: Capture running time to PostHog
if: github.run_attempt == '1'
continue-on-error: true
uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0
with:
posthog-token: ${{ secrets.POSTHOG_API_TOKEN }}
event: 'posthog-ci-running-time'
capture-run-duration: true
capture-job-durations: true
github-token: ${{ steps.telemetry-app-token.outputs.token }}
status-job: 'Visual regression tests pass'
- name: Capture running time to DevEx PostHog
if: github.run_attempt == '1'
continue-on-error: true
uses: PostHog/posthog-github-action@58dea254b598fb5d469c0699c98af8288a7f7650 # v1.2.0
with:
posthog-token: ${{ secrets.POSTHOG_DEVEX_PROJECT_API_TOKEN }}
event: 'posthog-ci-running-time'
capture-run-duration: true
capture-job-durations: true
github-token: ${{ steps.telemetry-app-token.outputs.token }}
status-job: 'Visual regression tests pass'
# Shadow measurement: reports which stories could have rendered differently
# Pick the visual-regression matrix on draft PRs based on which stories the
# webpack module graph (emitted by ModuleGraphPlugin) says are affected
# by the diff. On ready PRs and master push this job is skipped and
# visual-regression falls back to needs.changes.outputs.matrix (the full
# suite). The ready-for-review full run is the merge gate — storybook
# already skips merge_group entirely (changes.if). The run-ci-frontend
# label forces the full matrix on a draft. When selection can't be trusted
# on a draft, visual-regression is skipped and deferred to the ready run.
select-stories:
name: Story selection
runs-on: ubuntu-latest
timeout-minutes: 5
needs: [changes, build-storybook]
if: |
always() && needs.changes.outputs.frontend == 'true'
&& github.event_name == 'pull_request'
&& github.event.pull_request.draft == true
&& !contains(github.event.pull_request.labels.*.name, 'run-ci-frontend')
outputs:
mode: ${{ steps.classify.outputs.mode }}
matrix: ${{ steps.classify.outputs.matrix }}
should_run: ${{ steps.classify.outputs.should_run }}
affected_files: ${{ steps.classify.outputs.affected_files }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
filter: blob:none
fetch-depth: 500
ref: ${{ github.event.pull_request.head.sha }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: .nvmrc
token: ${{ github.token }}
- name: Download module-graph artifact
id: download
continue-on-error: true
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
name: storybook-module-graph
path: /tmp/storybook-graph
- name: Compute affected stories
id: selection
continue-on-error: true
env:
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
# Fetch the *current* tip of the base branch, not pull_request.base.sha —
# base.sha is captured at webhook time and goes stale if the branch later
# merges a newer master, which makes `base.sha...HEAD` balloon to include
# every merged-in master change.
git fetch --no-tags origin "$BASE_REF"
echo "### Changed files" >&2
git diff --name-only "origin/$BASE_REF"...HEAD >&2
RESULT=$(bin/find-affected-stories \
--graph /tmp/storybook-graph/module-graph.json \
--base-ref "origin/$BASE_REF")
echo "$RESULT" > /tmp/selection-result.json
# Stash compact form in step output (single line) for the summary step.
COMPACT=$(node -e 'const d = JSON.parse(require("fs").readFileSync("/tmp/selection-result.json", "utf8")); delete d.affected_stories; delete d.unresolved_changed_files; process.stdout.write(JSON.stringify(d))')
echo "result=$COMPACT" >> "$GITHUB_OUTPUT"
- name: Build narrowed matrix
id: classify
env:
SELECT_OUTCOME: ${{ steps.selection.outcome }}
run: |
set -euo pipefail
# Untrusted selection on a draft skips visual-regression entirely
# (should_run=false gates vr-setup and the test matrix off); the
# ready-for-review full run is the gate. A skipped draft can't
# trade green-by-default for green-by-skipping: drafts can't
# merge, and ready re-runs the full suite. The run-ci-frontend
# label forces the full matrix on a draft.
fall_back_to_skip() {
echo "mode=skip" >> "$GITHUB_OUTPUT"
echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT"
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "affected_files=" >> "$GITHUB_OUTPUT"
}
if [[ "$SELECT_OUTCOME" != "success" ]] || [[ ! -s /tmp/selection-result.json ]]; then
echo "::warning::story selector did not produce output; draft skips visual-regression (full run happens on ready for review)"
fall_back_to_skip
exit 0
fi
MODE=$(jq -r '.mode' /tmp/selection-result.json)
if [[ "$MODE" != "selective" ]]; then
echo "Selector declined to narrow (mode=$MODE); draft skips visual-regression (full run happens on ready for review)"
fall_back_to_skip
exit 0
fi
COUNT=$(jq -r '.affected_story_count' /tmp/selection-result.json)
if [[ "$COUNT" -eq 0 ]]; then
# Selective mode found zero affected stories — skip visual-regression entirely.
echo "mode=selective" >> "$GITHUB_OUTPUT"
echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT"
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "affected_files=" >> "$GITHUB_OUTPUT"
echo "Selective mode: 0 affected stories, skipping visual-regression"
exit 0
fi
# Selective with affected stories: collapse to one chromium shard.
AFFECTED_FILES=$(jq -r '.affected_stories | join(" ")' /tmp/selection-result.json)
echo "mode=selective" >> "$GITHUB_OUTPUT"
echo 'matrix={"include":[{"browser":"chromium","shard":1,"shard_count":1}]}' >> "$GITHUB_OUTPUT"
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "affected_files=$AFFECTED_FILES" >> "$GITHUB_OUTPUT"
echo "Selective mode: $COUNT affected story files"
- name: Write summary
if: steps.selection.outcome == 'success'
env:
RESULT: ${{ steps.selection.outputs.result }}
CURRENT_SHARDS: 16
SHARD_OVERHEAD_MINUTES: 3
run: |
MODE=$(echo "$RESULT" | jq -r '.mode')
TOTAL=$(echo "$RESULT" | jq -r '.total_story_count')
if [ "$MODE" = "selective" ]; then
COUNT=$(echo "$RESULT" | jq -r '.affected_story_count')
AFFECTED_DUR=$(echo "$RESULT" | jq -r '.affected_duration_seconds')
TOTAL_DUR=$(echo "$RESULT" | jq -r '.total_duration_seconds')
# Selected mode collapses to a single chromium shard (see classify step).
SHARDS=1
SAVED_STORIES=$((TOTAL - COUNT))
SAVED_SHARDS=$((CURRENT_SHARDS - SHARDS))
FULL_MINUTES=$(( (CURRENT_SHARDS * SHARD_OVERHEAD_MINUTES) + (TOTAL_DUR / 60) ))
SELECTIVE_MINUTES=$(( (SHARDS * SHARD_OVERHEAD_MINUTES) + (AFFECTED_DUR / 60) ))
SAVED_MINUTES=$((FULL_MINUTES - SELECTIVE_MINUTES))
cat >> "$GITHUB_STEP_SUMMARY" << EOF
## Story selection
| Metric | Full run | Selective | Saved |
|---|---|---|---|
| Story files | $TOTAL | $COUNT | **$SAVED_STORIES** |
| Shards | $CURRENT_SHARDS | $SHARDS | **$SAVED_SHARDS** |
| Est. CI minutes | ~${FULL_MINUTES} | ~${SELECTIVE_MINUTES} | **~${SAVED_MINUTES}** |
| Est. render time | ${TOTAL_DUR}s | ${AFFECTED_DUR}s | **$((TOTAL_DUR - AFFECTED_DUR))s** |
> Predictor: webpack module graph from build-storybook. Master push runs the full suite.
EOF
echo "SELECTION_METRICS: $(echo "$RESULT" | jq -c '{mode, affected: .affected_story_count, total: .total_story_count, current_shards: '$CURRENT_SHARDS', affected_dur: .affected_duration_seconds, total_dur: .total_duration_seconds}')"
else
REASON=$(echo "$RESULT" | jq -r '.reason // "unknown"')
cat >> "$GITHUB_STEP_SUMMARY" << EOF
## Story selection
**Full run required:** $REASON
Total stories: $TOTAL
EOF
echo "SELECTION_METRICS: $(echo "$RESULT" | jq -c '{mode, reason, total: .total_story_count}')"
fi