Skip to content
Run Playwright Tests

Run Playwright Tests #39863

Sign in to view logs

Run Playwright Tests

Run Playwright Tests #39863

Workflow file for this run

.github/workflows/pr-playwright-tests.yml at b77e759
name: Run Playwright Tests
concurrency:
group: Run-Playwright-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
cancel-in-progress: true
on:
merge_group:
pull_request:
branches:
- main
- "release/**"
# NOTE: Intentionally no `paths:` filter. We always trigger and let the
# `changes` job below decide whether the real test matrix runs. This
# avoids the dual-workflow skip pattern where a `paths-ignore`'d skip
# workflow can race the real workflow's same-named status check under
# branch protection.
push:
tags:
- "v*.*.*"
# TODO: Remove this if we enable merge-queues for release branches.
branches:
- "release/**"
permissions:
contents: read
env:
# Test Environment Variables
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
GEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
GOOGLE_PSE_API_KEY: ${{ secrets.GOOGLE_PSE_API_KEY }}
GOOGLE_PSE_SEARCH_ENGINE_ID: ${{ secrets.GOOGLE_PSE_SEARCH_ENGINE_ID }}
# for federated slack tests
SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
SLACK_CLIENT_SECRET: ${{ secrets.SLACK_CLIENT_SECRET }}
# MCP OAuth tests. AUDIENCE and REQUIRED_SCOPES are identical for both
# providers (the real Okta org is configured to mint exactly these), so they
# stay here. The issuer, JWKS URI, and client/user credentials are
# provider-specific and set per-job by a "Configure MCP OAuth provider" step
# (the only place `secrets` resolve) — NOT here, so the Okta job's $GITHUB_ENV
# writes aren't clobbered by a workflow-level definition. The playwright-tests
# job always uses the self-hosted mock OIDC IdP
# (deployment/docker_compose/docker-compose.mcp-oauth-test.yml): no secrets,
# deterministic, fork-safe, and gates every PR + merge_group. The
# playwright-tests-oauth-okta job additionally exercises the real Okta org when
# OAuth-relevant paths change — required on same-repo PRs, informational on
# merge_group.
MOCK_OIDC_PORT: 8090
MCP_OAUTH_AUDIENCE: api://mcp
MCP_OAUTH_REQUIRED_SCOPES: mcp:use
# for MCP API Key tests
MCP_API_KEY: test-api-key-12345
MCP_API_KEY_TEST_PORT: 8005
MCP_API_KEY_TEST_URL: http://host.docker.internal:8005/mcp
MCP_API_KEY_SERVER_HOST: 0.0.0.0
MCP_API_KEY_SERVER_PUBLIC_HOST: host.docker.internal
# for MCP per-user API key (multi-field template) tests
MCP_PER_USER_KEY_TEST_PORT: 8007
MCP_PER_USER_KEY_TEST_URL: http://host.docker.internal:8007/mcp
MCP_PER_USER_KEY_REQUIRED_HEADER: X-Username
MCP_PER_USER_KEY_SERVER_HOST: 0.0.0.0
MCP_PER_USER_KEY_SERVER_PUBLIC_HOST: host.docker.internal
MOCK_LLM_RESPONSE: true
MCP_TEST_SERVER_PORT: 8004
MCP_TEST_SERVER_URL: http://host.docker.internal:8004/mcp
MCP_TEST_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
MCP_TEST_SERVER_BIND_HOST: 0.0.0.0
MCP_TEST_SERVER_PUBLIC_HOST: host.docker.internal
MCP_SERVER_HOST: 0.0.0.0
MCP_SERVER_PUBLIC_HOST: host.docker.internal
MCP_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
# Visual regression S3 bucket (shared across all jobs)
PLAYWRIGHT_S3_BUCKET: onyx-playwright-artifacts
jobs:
changes:
# Decides whether the playwright matrix runs. On pull_request / merge_group
# we use paths-filter; on push (tags, release branches) we default to
# `true` so everything runs.
runs-on: ubuntu-latest
timeout-minutes: 5
# paths-filter needs pull-requests:read to list PR files on private repos (no-op on public).
permissions:
contents: read
pull-requests: read
outputs:
playwright: ${{ steps.filter.outputs.playwright || 'true' }}
airgap: ${{ steps.filter.outputs.airgap || 'true' }}
# Whether this change touches MCP-OAuth-relevant code. Drives whether a
# same-repo PR validates the OAuth spec against the real Okta org instead of
# the mock IdP. Defaults to 'false' on events where the filter doesn't run.
mcp_oauth: ${{ steps.filter.outputs.mcp_oauth || 'false' }}
steps:
- name: Checkout code
if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
id: filter
if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
with:
filters: |
playwright:
- 'backend/**'
- 'web/**'
- 'deployment/docker_compose/**'
- 'docker-bake.hcl'
- 'pyproject.toml'
- 'uv.lock'
- '.github/workflows/pr-playwright-tests.yml'
- '.github/actions/setup-test-license/**'
- '.github/actions/login-ecr-pullthrough-cache/**'
- '.github/actions/dhi-base-images/**'
airgap:
- 'backend/Dockerfile'
- 'backend/Dockerfile.model_server'
- 'backend/alembic/**'
- 'backend/alembic_tenants/**'
- 'backend/model_server/**'
- 'backend/onyx/configs/model_configs.py'
- 'backend/onyx/db/engine/**'
- 'backend/onyx/db/search_settings.py'
- 'backend/onyx/document_index/**'
- 'backend/onyx/main.py'
- 'backend/onyx/natural_language_processing/search_nlp_models.py'
- 'backend/onyx/natural_language_processing/utils.py'
- 'backend/onyx/setup.py'
- 'backend/requirements/**'
- 'backend/shared_configs/**'
- 'backend/tests/airgap/**'
- 'deployment/docker_compose/docker-compose.yml'
- 'deployment/docker_compose/docker-compose.airgap-test.yml'
- 'deployment/docker_compose/docker-compose.airgap-tls-test.yml'
- 'deployment/docker_compose/docker-compose.template.yml'
- 'pyproject.toml'
- 'uv.lock'
- '.github/workflows/pr-airgap-deployment-tests.yml'
- '.github/workflows/pr-playwright-tests.yml'
- '.github/actions/build-model-server-image/**'
- '.github/actions/login-ecr-pullthrough-cache/**'
- '.github/actions/dhi-base-images/**'
mcp_oauth:
- 'backend/onyx/server/features/mcp/**'
- 'backend/tests/integration/mock_services/mcp_test_server/**'
- 'web/tests/e2e/mcp/**'
- 'web/tests/e2e/pages/ActionsPopover.ts'
- 'deployment/docker_compose/docker-compose.mcp-oauth-test.yml'
- '.github/workflows/pr-playwright-tests.yml'
build-web-image:
needs: changes
if: needs.changes.outputs.playwright == 'true'
runs-on:
[
runs-on,
runner=4cpu-linux-arm64,
"run-id=${{ github.run_id }}-build-web-image",
"extras=ecr-cache",
]
timeout-minutes: 10
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Format branch name for cache
id: format-branch
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
REF_NAME: ${{ github.ref_name }}
run: |
if [ -n "${PR_NUMBER}" ]; then
CACHE_SUFFIX="${PR_NUMBER}"
else
# shellcheck disable=SC2001
CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
fi
echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
# web/Dockerfile defaults to the public Node bases. CI builds ship on the hardened
# DHI equivalents, passed as build args below.
- name: Resolve Docker Hardened Image bases
uses: ./.github/actions/dhi-base-images
with:
docker-username: ${{ secrets.DOCKER_USERNAME }}
docker-token: ${{ secrets.DOCKER_TOKEN }}
# SKIP_TYPE_CHECK cuts the build time of this image. Types are still checked
# by the `typescript-check` prek hook in the Quality Checks PR workflow.
- name: Build and push Web Docker image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: ./web
file: ./web/Dockerfile
platforms: linux/arm64
tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-web-${{ github.run_id }}
push: true
# Attestations attach as ECR referrers to the image digest, which is
# stable across runs and caps out at 100 per subject.
provenance: false
sbom: false
build-args: |
BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}
${{ env.DHI_NODE_BUILD_ARGS }}
SKIP_TYPE_CHECK=1
cache-from: |
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha || github.sha }}
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }}
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache
type=registry,ref=${{ env.BASE_IMAGE_REGISTRY }}/onyxdotapp/onyx-web-server:latest
cache-to: |
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache,mode=max
no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
build-backend-image:
needs: changes
if: >-
needs.changes.outputs.playwright == 'true' ||
needs.changes.outputs.airgap == 'true'
runs-on:
[
runs-on,
runner=1cpu-linux-arm64,
"run-id=${{ github.run_id }}-build-backend-image",
"extras=ecr-cache",
]
timeout-minutes: 15
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Format branch name for cache
id: format-branch
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
REF_NAME: ${{ github.ref_name }}
run: |
if [ -n "${PR_NUMBER}" ]; then
CACHE_SUFFIX="${PR_NUMBER}"
else
# shellcheck disable=SC2001
CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g')
fi
echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
- name: Build and push Backend Docker image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
with:
context: ./backend
file: ./backend/Dockerfile
# The production image; the Dockerfile's default (last) stage is the dev variant.
target: runtime
platforms: linux/arm64
tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-backend-${{ github.run_id }}
push: true
# Attestations attach as ECR referrers to the image digest, which is
# stable across runs and caps out at 100 per subject.
provenance: false
sbom: false
build-args: |
BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}
cache-from: |
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }}
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache
type=registry,ref=${{ env.BASE_IMAGE_REGISTRY }}/onyxdotapp/onyx-backend:latest
cache-to: |
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max
no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}
build-model-server-image:
needs: changes
if: >-
needs.changes.outputs.playwright == 'true' ||
needs.changes.outputs.airgap == 'true'
runs-on:
[
runs-on,
runner=1cpu-linux-arm64,
"run-id=${{ github.run_id }}-build-model-server-image",
"extras=ecr-cache",
]
timeout-minutes: 10
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Build model server image
uses: ./.github/actions/build-model-server-image
with:
runs-on-ecr-cache: ${{ env.RUNS_ON_ECR_CACHE }}
ref-name: ${{ github.ref_name }}
pr-number: ${{ github.event.pull_request.number }}
github-sha: ${{ github.event.pull_request.head.sha || github.sha }}
run-id: ${{ github.run_id }}
tag-prefix: playwright-test-model-server
platforms: linux/arm64
ecr-registry: ${{ vars.ECR_REGISTRY }}
docker-username: ${{ secrets.DOCKER_USERNAME }}
docker-token: ${{ secrets.DOCKER_TOKEN }}
airgap-health:
needs: [changes, build-backend-image, build-model-server-image]
if: needs.changes.outputs.airgap == 'true'
runs-on:
[
runs-on,
runner=4cpu-linux-arm64,
"run-id=${{ github.run_id }}-airgap-health",
"extras=ecr-cache",
]
timeout-minutes: 20
environment: ci-protected
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
with:
persist-credentials: false
- name: Set deployment images
env:
BACKEND_IMAGE: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-backend-${{ github.run_id }}
MODEL_SERVER_IMAGE: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-model-server-${{ github.run_id }}
run: |
echo "ONYX_BACKEND_IMAGE=${BACKEND_IMAGE}" >> "${GITHUB_ENV}"
echo "ONYX_MODEL_SERVER_IMAGE=${MODEL_SERVER_IMAGE}" >> "${GITHUB_ENV}"
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
- name: Pull deployment images
working-directory: deployment/docker_compose
run: |
docker pull "${ONYX_BACKEND_IMAGE}"
docker pull "${ONYX_MODEL_SERVER_IMAGE}"
docker compose -f docker-compose.yml pull relational_db opensearch cache minio
- name: Verify deployment without external network
working-directory: deployment/docker_compose
run: |
compose=(
docker compose
-p onyx-airgap
--profile s3-filestore
-f docker-compose.yml
-f docker-compose.airgap-test.yml
)
api_server_is_healthy() {
local container_id
container_id="$("${compose[@]}" ps -q api_server)"
[[ -n "${container_id}" ]] &&
[[ "$(docker inspect --format '{{.State.Health.Status}}' "${container_id}")" == "healthy" ]]
}
default_model_embeddings_are_finite() {
local container_id
container_id="$("${compose[@]}" ps -q api_server)"
[[ -n "${container_id}" ]] &&
docker exec -i "${container_id}" python - \
< ../../backend/tests/airgap/test_default_model_server_embeddings_are_finite.py
}
if ! "${compose[@]}" up -d --no-build --wait --wait-timeout 180 \
api_server inference_model_server minio ||
! api_server_is_healthy ||
! default_model_embeddings_are_finite; then
"${compose[@]}" logs --no-color > "${RUNNER_TEMP}/airgap.log"
exit 1
fi
"${compose[@]}" logs --no-color > "${RUNNER_TEMP}/airgap.log"
"${compose[@]}" down -v
- name: Verify deployment with untrusted Hugging Face TLS
working-directory: deployment/docker_compose
env:
AIRGAP_HF_ENDPOINT: https://tls_failure:4443
run: |
compose=(
docker compose
-p onyx-airgap-tls
--profile s3-filestore
-f docker-compose.yml
-f docker-compose.airgap-test.yml
-f docker-compose.airgap-tls-test.yml
)
api_server_is_healthy() {
local container_id
container_id="$("${compose[@]}" ps -q api_server)"
[[ -n "${container_id}" ]] &&
[[ "$(docker inspect --format '{{.State.Health.Status}}' "${container_id}")" == "healthy" ]]
}
if ! "${compose[@]}" up -d --no-build --wait --wait-timeout 180 \
api_server inference_model_server minio ||
! api_server_is_healthy; then
"${compose[@]}" logs --no-color > "${RUNNER_TEMP}/airgap-tls.log"
exit 1
fi
"${compose[@]}" logs --no-color > "${RUNNER_TEMP}/airgap-tls.log"
"${compose[@]}" down -v
- name: Stop deployments
if: always()
working-directory: deployment/docker_compose
run: |
docker compose -p onyx-airgap \
-f docker-compose.yml \
-f docker-compose.airgap-test.yml \
down -v || true
docker compose -p onyx-airgap-tls \
-f docker-compose.yml \
-f docker-compose.airgap-test.yml \
-f docker-compose.airgap-tls-test.yml \
down -v || true
- name: Upload deployment logs
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: airgap-deployment-logs
path: |
${{ runner.temp }}/airgap.log
${{ runner.temp }}/airgap-tls.log
playwright-tests:
needs: [build-web-image, build-backend-image, build-model-server-image]
name: Playwright Tests (${{ matrix.project }} ${{ matrix.shard }}/${{ matrix.shards }})
permissions:
id-token: write # Required for OIDC-based AWS credential exchange (S3 access for the dev license)
contents: read
runs-on:
- runs-on
- runner=8cpu-linux-arm64
- "run-id=${{ github.run_id }}-playwright-tests-${{ matrix.project }}-${{ matrix.shard }}"
- "extras=ecr-cache"
- volume=50gb
timeout-minutes: 20
strategy:
fail-fast: false
# Each entry is one parallel runner. `admin` (the bulk of the suite) is
# split into shards via Playwright's `--shard=index/total`; the smaller
# `exclusive` project runs as a single shard (1/1). Visual-regression
# screenshots are merged back together per-project in the
# `visual-regression` job below, so individual shards only ever hold a
# subset of screenshots.
matrix:
include:
- project: admin
shard: 1
shards: 2
- project: admin
shard: 2
shards: 2
- project: exclusive
shard: 1
shards: 1
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Setup bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # ratchet:oven-sh/setup-bun@v2 # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
with:
bun-version: "1.3.13"
- name: Install node dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Cache playwright cache
# zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-bun-${{ hashFiles('web/bun.lock') }}
restore-keys: |
${{ runner.os }}-playwright-bun-
- name: Install playwright browsers
working-directory: ./web
run: bunx playwright install chromium
- name: Setup test license
uses: ./.github/actions/setup-test-license
with:
aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}
- name: Create .env file for Docker Compose
env:
OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
RUN_ID: ${{ github.run_id }}
run: |
cat <<EOF > deployment/docker_compose/.env
COMPOSE_PROFILES=s3-filestore
ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
AUTH_TYPE=basic
INTEGRATION_TESTS_MODE=true
GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
EXA_API_KEY=${EXA_API_KEY_VALUE}
REQUIRE_EMAIL_VERIFICATION=false
DISABLE_TELEMETRY=true
# MCP E2E tests register mock servers on host.docker.internal (a private
# host-gateway address); opt in so the SSRF guard allows it. Cloud-
# metadata/link-local stays blocked regardless.
MCP_SERVER_ALLOW_PRIVATE_NETWORK=true
ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:playwright-test-model-server-${RUN_ID}
ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
EOF
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
# The MCP OAuth test drives the mock OIDC IdP in the browser at its
# advertised issuer (http://host.docker.internal:8090). The dockerized
# services reach that host via host-gateway, but the Playwright browser
# runs on the runner host, which doesn't resolve host.docker.internal by
# default — map it to the loopback where the IdP container publishes 8090.
- name: Make host.docker.internal resolvable for the browser
run: |
if ! grep -q "host.docker.internal" /etc/hosts; then
echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts
fi
# The main suite always runs the OAuth spec against the self-hosted mock
# OIDC IdP — deterministic, fork-safe, and the gate for every PR +
# merge_group. The real Okta org is validated separately by the
# playwright-tests-oauth-okta job. Written to $GITHUB_ENV (not the
# workflow-level env:) so that job can set its own values without a
# workflow-level definition clobbering them. getMcpOAuthConfig() requires
# the client/user vars even though the auto-issuing IdP ignores them.
- name: Configure MCP OAuth provider (mock IdP)
run: |
{
echo "MCP_OAUTH_ISSUER=http://host.docker.internal:8090"
echo "MCP_OAUTH_JWKS_URI=http://host.docker.internal:8090/jwks"
echo "MCP_OAUTH_CLIENT_ID=mock-oauth-client"
echo "MCP_OAUTH_CLIENT_SECRET=mock-oauth-secret"
echo "MCP_OAUTH_USERNAME=mock-user@example.com"
echo "MCP_OAUTH_PASSWORD=mock-password"
} >> "$GITHUB_ENV"
- name: Start Docker containers
run: |
cd deployment/docker_compose
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml up -d --wait --wait-timeout 300
id: start_docker
- name: Seed dev license
run: |
docker exec -e ONYX_DEV_LICENSE onyx-api_server-1 \
python -m scripts.seed_dev_license
- name: Run Playwright tests
working-directory: ./web
env:
PROJECT: ${{ matrix.project }}
SHARD: ${{ matrix.shard }}
SHARDS: ${{ matrix.shards }}
run: |
bunx playwright test --project "${PROJECT}" --shard="${SHARD}/${SHARDS}"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
# Includes test results and trace.zip files
name: playwright-test-results-${{ matrix.project }}-shard-${{ matrix.shard }}-${{ github.run_id }}
path: ./web/output/playwright/
retention-days: 30
# Per-shard screenshots. These are merged back together per-project in the
# `visual-regression` job, which is where the actual diff/baseline work
# happens — a single shard only ever holds a subset of a project's
# screenshots, so it must not compare or upload baselines on its own.
- name: Upload screenshots
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: playwright-screenshots-${{ matrix.project }}-shard-${{ matrix.shard }}-${{ github.run_id }}
path: ./web/output/screenshots/
if-no-files-found: ignore
retention-days: 30
# save before stopping the containers so the logs can be captured
- name: Save Docker logs
if: success() || failure()
env:
WORKSPACE: ${{ github.workspace }}
run: |
cd deployment/docker_compose
# Pass the same -f overrides as `up` so the mock MCP/IdP services
# (defined only in the override files) are included in the logs.
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml logs > docker-compose.log
mv docker-compose.log ${WORKSPACE}/docker-compose.log
- name: Upload logs
if: success() || failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: docker-logs-${{ matrix.project }}-shard-${{ matrix.shard }}-${{ github.run_id }}
path: ${{ github.workspace }}/docker-compose.log
# Validates the MCP OAuth spec against the REAL Okta org (the mock IdP run in
# playwright-tests is a deterministic stand-in). Runs only when OAuth-relevant
# paths change AND the secrets are reachable: always on merge_group, and on
# same-repo PRs (forks can't read secrets, so they rely on the mock-IdP run).
# Gating: this job's result is REQUIRED on pull_request and INFORMATIONAL on
# merge_group — see the playwright-required job.
playwright-tests-oauth-okta:
needs: [changes, build-web-image, build-backend-image, build-model-server-image]
if: >-
needs.changes.outputs.playwright == 'true' &&
needs.changes.outputs.mcp_oauth == 'true' && (
github.event_name == 'merge_group' || (
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name == github.repository
)
)
name: Playwright Tests (oauth-okta)
permissions:
id-token: write # Required for OIDC-based AWS credential exchange (license)
contents: read
runs-on:
- runs-on
- runner=8cpu-linux-arm64
- "run-id=${{ github.run_id }}-playwright-tests-oauth-okta"
- "extras=ecr-cache"
- volume=50gb
timeout-minutes: 15
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Setup bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # ratchet:oven-sh/setup-bun@v2 # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
with:
bun-version: "1.3.13"
- name: Install node dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Cache playwright cache
# zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-bun-${{ hashFiles('web/bun.lock') }}
restore-keys: |
${{ runner.os }}-playwright-bun-
- name: Install playwright browsers
working-directory: ./web
run: bunx playwright install chromium
- name: Setup test license
uses: ./.github/actions/setup-test-license
with:
aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}
- name: Create .env file for Docker Compose
env:
OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
RUN_ID: ${{ github.run_id }}
run: |
cat <<EOF > deployment/docker_compose/.env
COMPOSE_PROFILES=s3-filestore
ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
AUTH_TYPE=basic
INTEGRATION_TESTS_MODE=true
GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
EXA_API_KEY=${EXA_API_KEY_VALUE}
REQUIRE_EMAIL_VERIFICATION=false
DISABLE_TELEMETRY=true
# MCP E2E tests register mock servers on host.docker.internal (a private
# host-gateway address); opt in so the SSRF guard allows it. Cloud-
# metadata/link-local stays blocked regardless.
MCP_SERVER_ALLOW_PRIVATE_NETWORK=true
ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:playwright-test-model-server-${RUN_ID}
ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
EOF
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
- name: Make host.docker.internal resolvable for the browser
run: |
if ! grep -q "host.docker.internal" /etc/hosts; then
echo "127.0.0.1 host.docker.internal" | sudo tee -a /etc/hosts
fi
# Point the OAuth spec + mcp_oauth_server at the real Okta org. Written to
# $GITHUB_ENV (the only level where `secrets` resolve). This job only runs
# when the secrets are expected to be present, so an empty value is a hard
# error rather than a silent fall-back. The McpOAuthFlow page object drives
# the Okta login form (its selectors are still in DEFAULT_*_SELECTORS).
- name: Configure MCP OAuth provider (real Okta)
env:
OKTA_ISSUER: ${{ secrets.MCP_OAUTH_ISSUER }}
OKTA_JWKS_URI: ${{ secrets.MCP_OAUTH_JWKS_URI }}
OKTA_CLIENT_ID: ${{ secrets.MCP_OAUTH_CLIENT_ID }}
OKTA_CLIENT_SECRET: ${{ secrets.MCP_OAUTH_CLIENT_SECRET }}
OKTA_USERNAME: ${{ vars.MCP_OAUTH_USERNAME }}
OKTA_PASSWORD: ${{ secrets.MCP_OAUTH_PASSWORD }}
run: |
if [ -z "${OKTA_ISSUER}" ] || [ -z "${OKTA_JWKS_URI}" ] || \
[ -z "${OKTA_CLIENT_ID}" ] || [ -z "${OKTA_CLIENT_SECRET}" ] || \
[ -z "${OKTA_USERNAME}" ] || [ -z "${OKTA_PASSWORD}" ]; then
echo "::error::The MCP_OAUTH_* Okta secrets/vars are required but one or more are empty. Confirm they still exist in repo settings (secrets: ISSUER/JWKS_URI/CLIENT_ID/CLIENT_SECRET/PASSWORD; var: USERNAME)."
exit 1
fi
{
echo "MCP_OAUTH_ISSUER=${OKTA_ISSUER}"
echo "MCP_OAUTH_JWKS_URI=${OKTA_JWKS_URI}"
echo "MCP_OAUTH_CLIENT_ID=${OKTA_CLIENT_ID}"
echo "MCP_OAUTH_CLIENT_SECRET=${OKTA_CLIENT_SECRET}"
echo "MCP_OAUTH_USERNAME=${OKTA_USERNAME}"
echo "MCP_OAUTH_PASSWORD=${OKTA_PASSWORD}"
} >> "$GITHUB_ENV"
- name: Start Docker containers
run: |
cd deployment/docker_compose
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml up -d --wait --wait-timeout 300
id: start_docker
- name: Seed dev license
run: |
docker exec -e ONYX_DEV_LICENSE onyx-api_server-1 \
python -m scripts.seed_dev_license
- name: Run Playwright OAuth spec (real Okta)
working-directory: ./web
run: bunx playwright test mcp_oauth_flow --project admin
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: playwright-test-results-oauth-okta-${{ github.run_id }}
path: ./web/output/playwright/
retention-days: 30
- name: Upload screenshots
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: playwright-screenshots-oauth-okta-${{ github.run_id }}
path: ./web/output/screenshots/
retention-days: 30
- name: Save Docker logs
if: success() || failure()
env:
WORKSPACE: ${{ github.workspace }}
run: |
cd deployment/docker_compose
docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml -f docker-compose.mcp-per-user-key-test.yml logs > docker-compose.log
mv docker-compose.log ${WORKSPACE}/docker-compose.log
- name: Upload logs
if: success() || failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: docker-logs-oauth-okta-${{ github.run_id }}
path: ${{ github.workspace }}/docker-compose.log
playwright-tests-lite:
needs: [build-web-image, build-backend-image]
name: Playwright Tests (lite)
permissions:
id-token: write # Required for OIDC-based AWS credential exchange
contents: read
runs-on:
- runs-on
- runner=4cpu-linux-arm64
- "run-id=${{ github.run_id }}-playwright-tests-lite"
- "extras=ecr-cache"
timeout-minutes: 10
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
- name: Setup bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # ratchet:oven-sh/setup-bun@v2 # zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
with:
bun-version: "1.3.13"
- name: Install node dependencies
working-directory: ./web
run: bun install --frozen-lockfile
- name: Cache playwright cache
# zizmor: ignore[cache-poisoning] ephemeral runners; no release artifacts
uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-bun-${{ hashFiles('web/bun.lock') }}
restore-keys: |
${{ runner.os }}-playwright-bun-
- name: Install playwright browsers
working-directory: ./web
run: bunx playwright install chromium
- name: Setup test license
uses: ./.github/actions/setup-test-license
with:
aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}
- name: Create .env file for Docker Compose
env:
OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
RUN_ID: ${{ github.run_id }}
run: |
cat <<EOF > deployment/docker_compose/.env
ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
AUTH_TYPE=basic
INTEGRATION_TESTS_MODE=true
GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
MOCK_LLM_RESPONSE=true
REQUIRE_EMAIL_VERIFICATION=false
DISABLE_TELEMETRY=true
# MCP E2E tests register mock servers on host.docker.internal (a private
# host-gateway address); opt in so the SSRF guard allows it. Cloud-
# metadata/link-local stays blocked regardless.
MCP_SERVER_ALLOW_PRIVATE_NETWORK=true
ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
EOF
- name: Log in to ECR pull-through cache
uses: ./.github/actions/login-ecr-pullthrough-cache
with:
ecr-registry: ${{ vars.ECR_REGISTRY }}
- name: Start Docker containers (lite)
run: |
cd deployment/docker_compose
docker compose -f docker-compose.yml -f docker-compose.onyx-lite.yml -f docker-compose.dev.yml up -d --wait
id: start_docker
- name: Seed dev license
run: |
docker exec -e ONYX_DEV_LICENSE onyx-api_server-1 \
python -m scripts.seed_dev_license
- name: Run Playwright tests (lite)
working-directory: ./web
run: bunx playwright test --project lite
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: playwright-test-results-lite-${{ github.run_id }}
path: ./web/output/playwright/
retention-days: 30
- name: Save Docker logs
if: success() || failure()
env:
WORKSPACE: ${{ github.workspace }}
run: |
cd deployment/docker_compose
docker compose logs > docker-compose.log
mv docker-compose.log ${WORKSPACE}/docker-compose.log
- name: Upload logs
if: success() || failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: docker-logs-lite-${{ github.run_id }}
path: ${{ github.workspace }}/docker-compose.log
# Per-project visual regression. Because the test matrix is sharded, each
# shard only captures a subset of a project's screenshots. This job merges
# every shard's screenshots back into one directory before running the diff
# against the S3 baseline and (on main/release) re-uploading baselines with
# --delete. Running compare / upload-baselines on a partial set would report
# spurious "removed" screenshots and, worse, let shards race each other and
# corrupt the baseline — so this work MUST happen here, on the complete set.
visual-regression:
needs: [playwright-tests]
name: Visual Regression (${{ matrix.project }})
permissions:
id-token: write # Required for OIDC-based AWS credential exchange (S3 access)
contents: read
if: >-
always() &&
needs.playwright-tests.result != 'cancelled' &&
needs.playwright-tests.result != 'skipped'
runs-on:
- runs-on
- runner=2cpu-linux-arm64
- "run-id=${{ github.run_id }}-visual-regression-${{ matrix.project }}"
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
project: [admin, exclusive]
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
with:
persist-credentials: false
# Pull every shard's screenshots for this project into one directory.
# Shards capture disjoint screenshot names, so merge-multiple is safe.
# download-artifact errors when zero artifacts match the pattern (e.g. a
# shard failed before capturing any screenshot, so its artifact was never
# uploaded). Tolerate that here and let the presence check below decide
# whether there's anything to diff, so an infra failure degrades to a
# no-op rather than turning this advisory job red.
- name: Download shard screenshots
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
pattern: playwright-screenshots-${{ matrix.project }}-shard-*-${{ github.run_id }}
path: web/output/screenshots/
merge-multiple: true
- name: Check for screenshots
id: screenshots
env:
PROJECT: ${{ matrix.project }}
run: |
if [ -d "web/output/screenshots/" ] && [ -n "$(ls -A web/output/screenshots/ 2>/dev/null)" ]; then
echo "present=true" >> "$GITHUB_OUTPUT"
else
echo "present=false" >> "$GITHUB_OUTPUT"
echo "No shard screenshots found for ${PROJECT} — skipping diff and baseline update."
fi
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c
with:
role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
aws-region: us-east-2
- name: Install the latest version of uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # ratchet:astral-sh/setup-uv@v9.0.0
with:
enable-cache: false
version: "0.11.25"
- name: Determine baseline revision
id: baseline-rev
env:
EVENT_NAME: ${{ github.event_name }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
GH_REF: ${{ github.ref }}
REF_NAME: ${{ github.ref_name }}
run: |
if [ "${EVENT_NAME}" = "pull_request" ]; then
# PRs compare against the base branch (e.g. main, release/2.5)
echo "rev=${BASE_REF}" >> "$GITHUB_OUTPUT"
elif [ "${EVENT_NAME}" = "merge_group" ]; then
# Merge queue compares against the target branch (e.g. refs/heads/main -> main)
echo "rev=${MERGE_GROUP_BASE_REF#refs/heads/}" >> "$GITHUB_OUTPUT"
elif [[ "${GH_REF}" == refs/tags/* ]]; then
# Tag builds compare against the tag name
echo "rev=${REF_NAME}" >> "$GITHUB_OUTPUT"
else
# Push builds (main, release/*) compare against the branch name
echo "rev=${REF_NAME}" >> "$GITHUB_OUTPUT"
fi
- name: Generate screenshot diff report
if: steps.screenshots.outputs.present == 'true'
env:
PROJECT: ${{ matrix.project }}
PLAYWRIGHT_S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
BASELINE_REV: ${{ steps.baseline-rev.outputs.rev }}
run: |
uv run --no-sync --with onyx-devtools ods screenshot-diff compare \
--project "${PROJECT}" \
--rev "${BASELINE_REV}"
# PR-only: PR_NUMBER is empty on push/merge_group events, which would write
# the report under a junk `reports/pr-/...` key. Baselines (updated below)
# are the meaningful artifact on those events, not this PR-scoped report.
- name: Upload visual diff report to S3
if: github.event_name == 'pull_request' && steps.screenshots.outputs.present == 'true'
env:
PROJECT: ${{ matrix.project }}
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ github.run_id }}
run: |
SUMMARY_FILE="web/output/screenshot-diff/${PROJECT}/summary.json"
if [ ! -f "${SUMMARY_FILE}" ]; then
echo "No summary file found — skipping S3 upload."
exit 0
fi
HAS_DIFF=$(jq -r '.has_differences' "${SUMMARY_FILE}")
if [ "${HAS_DIFF}" != "true" ]; then
echo "No visual differences for ${PROJECT} — skipping S3 upload."
exit 0
fi
aws s3 sync "web/output/screenshot-diff/${PROJECT}/" \
"s3://${PLAYWRIGHT_S3_BUCKET}/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/"
- name: Upload visual diff summary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: screenshot-diff-summary-${{ matrix.project }}
path: ./web/output/screenshot-diff/${{ matrix.project }}/summary.json
if-no-files-found: ignore
retention-days: 5
- name: Upload visual diff report artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
if: always()
with:
name: screenshot-diff-report-${{ matrix.project }}-${{ github.run_id }}
path: ./web/output/screenshot-diff/${{ matrix.project }}/
if-no-files-found: ignore
retention-days: 30
- name: Update S3 baselines
# Only update baselines from a fully green run on a protected ref.
# `success()` covers the diff steps above; `needs.playwright-tests.result`
# ensures no shard (in any project) failed before we treat this run's
# screenshots as the new source of truth.
if: >-
success() &&
needs.playwright-tests.result == 'success' && (
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release/') ||
startsWith(github.ref, 'refs/tags/v') ||
(
github.event_name == 'merge_group' && (
github.event.merge_group.base_ref == 'refs/heads/main' ||
startsWith(github.event.merge_group.base_ref, 'refs/heads/release/')
)
)
)
env:
PROJECT: ${{ matrix.project }}
PLAYWRIGHT_S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
BASELINE_REV: ${{ steps.baseline-rev.outputs.rev }}
run: |
if [ -d "web/output/screenshots/" ] && [ "$(ls -A web/output/screenshots/)" ]; then
uv run --no-sync --with onyx-devtools ods screenshot-diff upload-baselines \
--project "${PROJECT}" \
--rev "${BASELINE_REV}" \
--delete
else
echo "No screenshots to upload for ${PROJECT} — skipping baseline update."
fi
# Post a single combined visual regression comment after all matrix jobs finish
visual-regression-comment:
needs: [visual-regression]
if: >-
always() &&
github.event_name == 'pull_request' &&
needs.visual-regression.result != 'cancelled' &&
needs.visual-regression.result != 'skipped'
runs-on: ubuntu-slim
timeout-minutes: 5
permissions:
pull-requests: write
steps:
- name: Download visual diff summaries
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
pattern: screenshot-diff-summary-*
path: summaries/
- name: Post combined PR comment
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
RUN_ID: ${{ github.run_id }}
REPO: ${{ github.repository }}
S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
run: |
MARKER="<!-- visual-regression-report -->"
# Build the markdown table from all summary files
TABLE_HEADER="| Project | Changed | Added | Removed | Unchanged | Report |"
TABLE_DIVIDER="|---------|---------|-------|---------|-----------|--------|"
TABLE_ROWS=""
HAS_ANY_SUMMARY=false
for SUMMARY_DIR in summaries/screenshot-diff-summary-*/; do
SUMMARY_FILE="${SUMMARY_DIR}summary.json"
if [ ! -f "${SUMMARY_FILE}" ]; then
continue
fi
HAS_ANY_SUMMARY=true
PROJECT=$(jq -r '.project' "${SUMMARY_FILE}")
CHANGED=$(jq -r '.changed' "${SUMMARY_FILE}")
ADDED=$(jq -r '.added' "${SUMMARY_FILE}")
REMOVED=$(jq -r '.removed' "${SUMMARY_FILE}")
UNCHANGED=$(jq -r '.unchanged' "${SUMMARY_FILE}")
TOTAL=$(jq -r '.total' "${SUMMARY_FILE}")
HAS_DIFF=$(jq -r '.has_differences' "${SUMMARY_FILE}")
if [ "${TOTAL}" = "0" ]; then
REPORT_LINK="_No screenshots_"
elif [ "${HAS_DIFF}" = "true" ]; then
REPORT_URL="https://${S3_BUCKET}.s3.us-east-2.amazonaws.com/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/index.html"
REPORT_LINK="[View Report](${REPORT_URL})"
else
REPORT_LINK="✅ No changes"
fi
TABLE_ROWS="${TABLE_ROWS}| \`${PROJECT}\` | ${CHANGED} | ${ADDED} | ${REMOVED} | ${UNCHANGED} | ${REPORT_LINK} |\n"
done
if [ "${HAS_ANY_SUMMARY}" = "false" ]; then
echo "No visual diff summaries found — skipping PR comment."
exit 0
fi
BODY=$(printf '%s\n' \
"${MARKER}" \
"### 🖼️ Visual Regression Report" \
"" \
"${TABLE_HEADER}" \
"${TABLE_DIVIDER}" \
"$(printf '%b' "${TABLE_ROWS}")")
# Upsert: find existing comment with the marker, or create a new one
EXISTING_COMMENT_ID=$(gh api \
"repos/${REPO}/issues/${PR_NUMBER}/comments" \
--jq ".[] | select(.body | startswith(\"${MARKER}\")) | .id" \
2>/dev/null | head -1)
if [ -n "${EXISTING_COMMENT_ID}" ]; then
gh api \
--method PATCH \
"repos/${REPO}/issues/comments/${EXISTING_COMMENT_ID}" \
-f body="${BODY}"
else
gh api \
--method POST \
"repos/${REPO}/issues/${PR_NUMBER}/comments" \
-f body="${BODY}"
fi
playwright-required:
# NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
# This job is the single required status check for the playwright suite. It
# always runs so branch protection has something to wait on, and it passes
# cleanly when `changes` reports no relevant paths changed (i.e. the test
# matrix was legitimately skipped).
runs-on: ubuntu-slim
timeout-minutes: 5
needs:
[
changes,
airgap-health,
playwright-tests,
playwright-tests-lite,
playwright-tests-oauth-okta,
]
if: ${{ always() }}
steps:
- name: Check job status
env:
EVENT_NAME: ${{ github.event_name }}
CHANGES_RESULT: ${{ needs.changes.result }}
RUN_AIRGAP: ${{ needs.changes.outputs.airgap }}
RUN_TESTS: ${{ needs.changes.outputs.playwright }}
AIRGAP_RESULT: ${{ needs.airgap-health.result }}
TESTS_RESULT: ${{ needs.playwright-tests.result }}
LITE_RESULT: ${{ needs.playwright-tests-lite.result }}
OKTA_RESULT: ${{ needs.playwright-tests-oauth-okta.result }}
run: |
# Fail closed if `changes` didn't succeed. Otherwise an empty
# RUN_TESTS (which is what we'd see when `changes` failed/cancelled)
# would be indistinguishable from "no relevant paths changed" and we
# would incorrectly pass the required check.
if [ "${CHANGES_RESULT}" != "success" ]; then
echo "changes job did not succeed (result: ${CHANGES_RESULT})"
exit 1
fi
if [ "${RUN_AIRGAP}" = "true" ] && [ "${AIRGAP_RESULT}" != "success" ]; then
echo "Airgap deployment result: ${AIRGAP_RESULT}"
exit 1
fi
if [ "${RUN_TESTS}" != "true" ]; then
echo "No relevant paths changed — required check passes."
exit 0
fi
if [ "${TESTS_RESULT}" != "success" ] || [ "${LITE_RESULT}" != "success" ]; then
echo "Test results: playwright-tests=${TESTS_RESULT}, playwright-tests-lite=${LITE_RESULT}"
exit 1
fi
# The real-Okta OAuth job gates pull_request runs (blocks the PR), but
# is informational on merge_group so a flaky Okta run can't wedge the
# serialized merge queue. `skipped` means it wasn't applicable (no OAuth
# paths changed, or a fork PR) and is always acceptable.
if [ "${EVENT_NAME}" = "pull_request" ] && \
[ "${OKTA_RESULT}" != "success" ] && [ "${OKTA_RESULT}" != "skipped" ]; then
echo "Okta OAuth job result (required on PRs): ${OKTA_RESULT}"
exit 1
fi
echo "All tests passed."