Run Playwright Tests #20160

Workflow file for this run

.github/workflows/pr-playwright-tests.yml at 9f50b2f

	name: Run Playwright Tests
	concurrency:
	group: Run-Playwright-Tests-${{ github.workflow }}-${{ github.head_ref \|\| github.event.workflow_run.head_branch \|\| github.run_id }}
	cancel-in-progress: true

	on:
	merge_group:
	pull_request:
	branches:
	- main
	- "release/**"
	push:
	tags:
	- "v..*"

	permissions:
	contents: read

	env:
	# Test Environment Variables
	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
	GEN_AI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
	EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
	FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
	GOOGLE_PSE_API_KEY: ${{ secrets.GOOGLE_PSE_API_KEY }}
	GOOGLE_PSE_SEARCH_ENGINE_ID: ${{ secrets.GOOGLE_PSE_SEARCH_ENGINE_ID }}

	# for federated slack tests
	SLACK_CLIENT_ID: ${{ secrets.SLACK_CLIENT_ID }}
	SLACK_CLIENT_SECRET: ${{ secrets.SLACK_CLIENT_SECRET }}

	# for MCP Oauth tests
	MCP_OAUTH_CLIENT_ID: ${{ secrets.MCP_OAUTH_CLIENT_ID }}
	MCP_OAUTH_CLIENT_SECRET: ${{ secrets.MCP_OAUTH_CLIENT_SECRET }}
	MCP_OAUTH_ISSUER: ${{ secrets.MCP_OAUTH_ISSUER }}
	MCP_OAUTH_JWKS_URI: ${{ secrets.MCP_OAUTH_JWKS_URI }}
	MCP_OAUTH_USERNAME: ${{ vars.MCP_OAUTH_USERNAME }}
	MCP_OAUTH_PASSWORD: ${{ secrets.MCP_OAUTH_PASSWORD }}

	# for MCP API Key tests
	MCP_API_KEY: test-api-key-12345
	MCP_API_KEY_TEST_PORT: 8005
	MCP_API_KEY_TEST_URL: http://host.docker.internal:8005/mcp
	MCP_API_KEY_SERVER_HOST: 0.0.0.0
	MCP_API_KEY_SERVER_PUBLIC_HOST: host.docker.internal

	MOCK_LLM_RESPONSE: true
	MCP_TEST_SERVER_PORT: 8004
	MCP_TEST_SERVER_URL: http://host.docker.internal:8004/mcp
	MCP_TEST_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp
	MCP_TEST_SERVER_BIND_HOST: 0.0.0.0
	MCP_TEST_SERVER_PUBLIC_HOST: host.docker.internal
	MCP_SERVER_HOST: 0.0.0.0
	MCP_SERVER_PUBLIC_HOST: host.docker.internal
	MCP_SERVER_PUBLIC_URL: http://host.docker.internal:8004/mcp

	# Visual regression S3 bucket (shared across all jobs)
	PLAYWRIGHT_S3_BUCKET: onyx-playwright-artifacts

	jobs:
	build-web-image:
	runs-on:
	[
	runs-on,
	runner=4cpu-linux-arm64,
	"run-id=${{ github.run_id }}-build-web-image",
	"extras=ecr-cache",
	]
	timeout-minutes: 45
	steps:
	- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2

	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Format branch name for cache
	id: format-branch
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	REF_NAME: ${{ github.ref_name }}
	run: \|
	if [ -n "${PR_NUMBER}" ]; then
	CACHE_SUFFIX="${PR_NUMBER}"
	else
	# shellcheck disable=SC2001
	CACHE_SUFFIX=$(echo "${REF_NAME}" \| sed 's/[^A-Za-z0-9._-]/-/g')
	fi
	echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3

	# needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
	# https://docs.docker.com/docker-hub/usage/
	- name: Login to Docker Hub
	uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_TOKEN }}

	- name: Build and push Web Docker image
	uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
	with:
	context: ./web
	file: ./web/Dockerfile
	platforms: linux/arm64
	tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-web-${{ github.run_id }}
	push: true
	cache-from: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha \|\| github.sha }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache
	type=registry,ref=onyxdotapp/onyx-web-server:latest
	cache-to: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha \|\| github.sha }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache,mode=max
	no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}

	build-backend-image:
	runs-on:
	[
	runs-on,
	runner=1cpu-linux-arm64,
	"run-id=${{ github.run_id }}-build-backend-image",
	"extras=ecr-cache",
	]
	timeout-minutes: 45
	steps:
	- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2

	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Format branch name for cache
	id: format-branch
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	REF_NAME: ${{ github.ref_name }}
	run: \|
	if [ -n "${PR_NUMBER}" ]; then
	CACHE_SUFFIX="${PR_NUMBER}"
	else
	# shellcheck disable=SC2001
	CACHE_SUFFIX=$(echo "${REF_NAME}" \| sed 's/[^A-Za-z0-9._-]/-/g')
	fi
	echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3

	# needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
	# https://docs.docker.com/docker-hub/usage/
	- name: Login to Docker Hub
	uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_TOKEN }}

	- name: Build and push Backend Docker image
	uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
	with:
	context: ./backend
	file: ./backend/Dockerfile
	platforms: linux/arm64
	tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-backend-${{ github.run_id }}
	push: true
	cache-from: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha \|\| github.sha }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache
	type=registry,ref=onyxdotapp/onyx-backend:latest
	cache-to: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha \|\| github.sha }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max
	no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}

	build-model-server-image:
	runs-on:
	[
	runs-on,
	runner=1cpu-linux-arm64,
	"run-id=${{ github.run_id }}-build-model-server-image",
	"extras=ecr-cache",
	]
	timeout-minutes: 45
	steps:
	- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2

	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Format branch name for cache
	id: format-branch
	env:
	PR_NUMBER: ${{ github.event.pull_request.number }}
	REF_NAME: ${{ github.ref_name }}
	run: \|
	if [ -n "${PR_NUMBER}" ]; then
	CACHE_SUFFIX="${PR_NUMBER}"
	else
	# shellcheck disable=SC2001
	CACHE_SUFFIX=$(echo "${REF_NAME}" \| sed 's/[^A-Za-z0-9._-]/-/g')
	fi
	echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # ratchet:docker/setup-buildx-action@v3

	# needed for pulling external images otherwise, we hit the "Unauthenticated users" limit
	# https://docs.docker.com/docker-hub/usage/
	- name: Login to Docker Hub
	uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_TOKEN }}

	- name: Build and push Model Server Docker image
	uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6
	with:
	context: ./backend
	file: ./backend/Dockerfile.model_server
	platforms: linux/arm64
	tags: ${{ env.RUNS_ON_ECR_CACHE }}:playwright-test-model-server-${{ github.run_id }}
	push: true
	cache-from: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha \|\| github.sha }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }}
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache
	type=registry,ref=onyxdotapp/onyx-model-server:latest
	cache-to: \|
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha \|\| github.sha }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max
	type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max
	no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }}

	playwright-tests:
	needs: [build-web-image, build-backend-image, build-model-server-image]
	name: Playwright Tests (${{ matrix.project }})
	permissions:
	id-token: write # Required for OIDC-based AWS credential exchange (S3 access)
	contents: read
	runs-on:
	- runs-on
	- runner=8cpu-linux-arm64
	- "run-id=${{ github.run_id }}-playwright-tests-${{ matrix.project }}"
	- "extras=ecr-cache"
	- volume=50gb
	timeout-minutes: 45
	strategy:
	fail-fast: false
	matrix:
	project: [admin, exclusive]
	steps:
	- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2

	- name: Checkout code
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Setup node
	uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # ratchet:actions/setup-node@v4
	with:
	node-version: 22
	cache: "npm"
	cache-dependency-path: ./web/package-lock.json

	- name: Install node dependencies
	working-directory: ./web
	run: npm ci

	- name: Cache playwright cache
	uses: runs-on/cache@50350ad4242587b6c8c2baa2e740b1bc11285ff4 # ratchet:runs-on/cache@v4
	with:
	path: ~/.cache/ms-playwright
	key: ${{ runner.os }}-playwright-npm-${{ hashFiles('web/package-lock.json') }}
	restore-keys: \|
	${{ runner.os }}-playwright-npm-

	- name: Install playwright browsers
	working-directory: ./web
	run: npx playwright install --with-deps

	- name: Create .env file for Docker Compose
	env:
	OPENAI_API_KEY_VALUE: ${{ env.OPENAI_API_KEY }}
	EXA_API_KEY_VALUE: ${{ env.EXA_API_KEY }}
	ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
	RUN_ID: ${{ github.run_id }}
	run: \|
	cat <<EOF > deployment/docker_compose/.env
	COMPOSE_PROFILES=s3-filestore
	ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true
	# TODO(Nik): https://linear.app/onyx-app/issue/ENG-1/update-test-infra-to-use-test-license
	LICENSE_ENFORCEMENT_ENABLED=false
	AUTH_TYPE=basic
	INTEGRATION_TESTS_MODE=true
	GEN_AI_API_KEY=${OPENAI_API_KEY_VALUE}
	EXA_API_KEY=${EXA_API_KEY_VALUE}
	REQUIRE_EMAIL_VERIFICATION=false
	DISABLE_TELEMETRY=true
	ONYX_BACKEND_IMAGE=${ECR_CACHE}:playwright-test-backend-${RUN_ID}
	ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:playwright-test-model-server-${RUN_ID}
	ONYX_WEB_SERVER_IMAGE=${ECR_CACHE}:playwright-test-web-${RUN_ID}
	EOF

	# needed for pulling Vespa, Redis, Postgres, and Minio images
	# otherwise, we hit the "Unauthenticated users" limit
	# https://docs.docker.com/docker-hub/usage/
	- name: Login to Docker Hub
	uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # ratchet:docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_TOKEN }}

	- name: Start Docker containers
	run: \|
	cd deployment/docker_compose
	docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.mcp-oauth-test.yml -f docker-compose.mcp-api-key-test.yml up -d
	id: start_docker

	- name: Wait for service to be ready
	run: \|
	echo "Starting wait-for-service script..."

	docker logs -f onyx-api_server-1 &

	start_time=$(date +%s)
	timeout=300 # 5 minutes in seconds

	while true; do
	current_time=$(date +%s)
	elapsed_time=$((current_time - start_time))

	if [ $elapsed_time -ge $timeout ]; then
	echo "Timeout reached. Service did not become ready in 5 minutes."
	exit 1
	fi

	# Use curl with error handling to ignore specific exit code 56
	response=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health \|\| echo "curl_error")

	if [ "$response" = "200" ]; then
	echo "Service is ready!"
	break
	elif [ "$response" = "curl_error" ]; then
	echo "Curl encountered an error, possibly exit code 56. Continuing to retry..."
	else
	echo "Service not ready yet (HTTP status $response). Retrying in 5 seconds..."
	fi

	sleep 5
	done
	echo "Finished waiting for service."

	- name: Wait for MCP OAuth mock server
	run: \|
	echo "Waiting for MCP OAuth mock server on port ${MCP_TEST_SERVER_PORT:-8004}..."
	start_time=$(date +%s)
	timeout=120

	while true; do
	current_time=$(date +%s)
	elapsed_time=$((current_time - start_time))

	if [ $elapsed_time -ge $timeout ]; then
	echo "Timeout reached. MCP OAuth mock server did not become ready in ${timeout}s."
	exit 1
	fi

	if curl -sf "http://localhost:${MCP_TEST_SERVER_PORT:-8004}/healthz" > /dev/null; then
	echo "MCP OAuth mock server is ready!"
	break
	fi

	sleep 3
	done

	- name: Wait for MCP API Key mock server
	run: \|
	echo "Waiting for MCP API Key mock server on port ${MCP_API_KEY_TEST_PORT:-8005}..."
	start_time=$(date +%s)
	timeout=120

	while true; do
	current_time=$(date +%s)
	elapsed_time=$((current_time - start_time))

	if [ $elapsed_time -ge $timeout ]; then
	echo "Timeout reached. MCP API Key mock server did not become ready in ${timeout}s."
	exit 1
	fi

	if curl -sf "http://localhost:${MCP_API_KEY_TEST_PORT:-8005}/healthz" > /dev/null; then
	echo "MCP API Key mock server is ready!"
	break
	fi

	sleep 3
	done

	- name: Wait for web server to be ready
	run: \|
	echo "Waiting for web server on port 3000..."
	start_time=$(date +%s)
	timeout=120

	while true; do
	current_time=$(date +%s)
	elapsed_time=$((current_time - start_time))

	if [ $elapsed_time -ge $timeout ]; then
	echo "Timeout reached. Web server did not become ready in ${timeout}s."
	exit 1
	fi

	if curl -sf "http://localhost:3000/api/health" > /dev/null 2>&1 \|\| \
	curl -sf "http://localhost:3000/" > /dev/null 2>&1; then
	echo "Web server is ready!"
	break
	fi

	echo "Web server not ready yet. Retrying in 3 seconds..."
	sleep 3
	done

	- name: Run Playwright tests
	working-directory: ./web
	env:
	PROJECT: ${{ matrix.project }}
	run: \|
	npx playwright test --project ${PROJECT}

	- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
	if: always()
	with:
	# Includes test results and trace.zip files
	name: playwright-test-results-${{ matrix.project }}-${{ github.run_id }}
	path: ./web/output/playwright/
	retention-days: 30

	- name: Upload screenshots
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
	if: always()
	with:
	name: playwright-screenshots-${{ matrix.project }}-${{ github.run_id }}
	path: ./web/output/screenshots/
	retention-days: 30

	# --- Visual Regression Diff ---
	- name: Configure AWS credentials
	if: always()
	uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708
	with:
	role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
	aws-region: us-east-2

	- name: Install the latest version of uv
	if: always()
	uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # ratchet:astral-sh/setup-uv@v7
	with:
	enable-cache: false
	version: "0.9.9"

	- name: Determine baseline revision
	if: always()
	id: baseline-rev
	env:
	EVENT_NAME: ${{ github.event_name }}
	BASE_REF: ${{ github.event.pull_request.base.ref }}
	MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
	GH_REF: ${{ github.ref }}
	REF_NAME: ${{ github.ref_name }}
	run: \|
	if [ "${EVENT_NAME}" = "pull_request" ]; then
	# PRs compare against the base branch (e.g. main, release/2.5)
	echo "rev=${BASE_REF}" >> "$GITHUB_OUTPUT"
	elif [ "${EVENT_NAME}" = "merge_group" ]; then
	# Merge queue compares against the target branch (e.g. refs/heads/main -> main)
	echo "rev=${MERGE_GROUP_BASE_REF#refs/heads/}" >> "$GITHUB_OUTPUT"
	elif [[ "${GH_REF}" == refs/tags/* ]]; then
	# Tag builds compare against the tag name
	echo "rev=${REF_NAME}" >> "$GITHUB_OUTPUT"
	else
	# Push builds (main, release/*) compare against the branch name
	echo "rev=${REF_NAME}" >> "$GITHUB_OUTPUT"
	fi

	- name: Generate screenshot diff report
	if: always()
	env:
	PROJECT: ${{ matrix.project }}
	PLAYWRIGHT_S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
	BASELINE_REV: ${{ steps.baseline-rev.outputs.rev }}
	run: \|
	uv run --no-sync --with onyx-devtools ods screenshot-diff compare \
	--project "${PROJECT}" \
	--rev "${BASELINE_REV}"

	- name: Upload visual diff report to S3
	if: always()
	env:
	PROJECT: ${{ matrix.project }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	RUN_ID: ${{ github.run_id }}
	run: \|
	SUMMARY_FILE="web/output/screenshot-diff/${PROJECT}/summary.json"
	if [ ! -f "${SUMMARY_FILE}" ]; then
	echo "No summary file found — skipping S3 upload."
	exit 0
	fi

	HAS_DIFF=$(jq -r '.has_differences' "${SUMMARY_FILE}")
	if [ "${HAS_DIFF}" != "true" ]; then
	echo "No visual differences for ${PROJECT} — skipping S3 upload."
	exit 0
	fi

	aws s3 sync "web/output/screenshot-diff/${PROJECT}/" \
	"s3://${PLAYWRIGHT_S3_BUCKET}/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/"

	- name: Upload visual diff summary
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
	if: always()
	with:
	name: screenshot-diff-summary-${{ matrix.project }}
	path: ./web/output/screenshot-diff/${{ matrix.project }}/summary.json
	if-no-files-found: ignore
	retention-days: 5

	- name: Upload visual diff report artifact
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
	if: always()
	with:
	name: screenshot-diff-report-${{ matrix.project }}-${{ github.run_id }}
	path: ./web/output/screenshot-diff/${{ matrix.project }}/
	if-no-files-found: ignore
	retention-days: 30

	- name: Update S3 baselines
	if: >-
	success() && (
	github.ref == 'refs/heads/main' \|\|
	startsWith(github.ref, 'refs/heads/release/') \|\|
	startsWith(github.ref, 'refs/tags/v') \|\|
	(
	github.event_name == 'merge_group' && (
	github.event.merge_group.base_ref == 'refs/heads/main' \|\|
	startsWith(github.event.merge_group.base_ref, 'refs/heads/release/')
	)
	)
	)
	env:
	PROJECT: ${{ matrix.project }}
	PLAYWRIGHT_S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
	BASELINE_REV: ${{ steps.baseline-rev.outputs.rev }}
	run: \|
	if [ -d "web/output/screenshots/" ] && [ "$(ls -A web/output/screenshots/)" ]; then
	uv run --no-sync --with onyx-devtools ods screenshot-diff upload-baselines \
	--project "${PROJECT}" \
	--rev "${BASELINE_REV}" \
	--delete
	else
	echo "No screenshots to upload for ${PROJECT} — skipping baseline update."
	fi

	# save before stopping the containers so the logs can be captured
	- name: Save Docker logs
	if: success() \|\| failure()
	env:
	WORKSPACE: ${{ github.workspace }}
	run: \|
	cd deployment/docker_compose
	docker compose logs > docker-compose.log
	mv docker-compose.log ${WORKSPACE}/docker-compose.log

	- name: Upload logs
	if: success() \|\| failure()
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
	with:
	name: docker-logs-${{ matrix.project }}-${{ github.run_id }}
	path: ${{ github.workspace }}/docker-compose.log

	# Post a single combined visual regression comment after all matrix jobs finish
	visual-regression-comment:
	needs: [playwright-tests]
	if: >-
	always() &&
	github.event_name == 'pull_request' &&
	needs.playwright-tests.result != 'cancelled'
	runs-on: ubuntu-slim
	timeout-minutes: 5
	permissions:
	pull-requests: write
	steps:
	- name: Download visual diff summaries
	uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # ratchet:actions/download-artifact@v4
	with:
	pattern: screenshot-diff-summary-*
	path: summaries/

	- name: Post combined PR comment
	env:
	GH_TOKEN: ${{ github.token }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	RUN_ID: ${{ github.run_id }}
	REPO: ${{ github.repository }}
	S3_BUCKET: ${{ env.PLAYWRIGHT_S3_BUCKET }}
	run: \|
	MARKER="<!-- visual-regression-report -->"

	# Build the markdown table from all summary files
	TABLE_HEADER="\| Project \| Changed \| Added \| Removed \| Unchanged \| Report \|"
	TABLE_DIVIDER="\|---------\|---------\|-------\|---------\|-----------\|--------\|"
	TABLE_ROWS=""
	HAS_ANY_SUMMARY=false

	for SUMMARY_DIR in summaries/screenshot-diff-summary-*/; do
	SUMMARY_FILE="${SUMMARY_DIR}summary.json"
	if [ ! -f "${SUMMARY_FILE}" ]; then
	continue
	fi

	HAS_ANY_SUMMARY=true
	PROJECT=$(jq -r '.project' "${SUMMARY_FILE}")
	CHANGED=$(jq -r '.changed' "${SUMMARY_FILE}")
	ADDED=$(jq -r '.added' "${SUMMARY_FILE}")
	REMOVED=$(jq -r '.removed' "${SUMMARY_FILE}")
	UNCHANGED=$(jq -r '.unchanged' "${SUMMARY_FILE}")
	TOTAL=$(jq -r '.total' "${SUMMARY_FILE}")
	HAS_DIFF=$(jq -r '.has_differences' "${SUMMARY_FILE}")

	if [ "${TOTAL}" = "0" ]; then
	REPORT_LINK="_No screenshots_"
	elif [ "${HAS_DIFF}" = "true" ]; then
	REPORT_URL="https://${S3_BUCKET}.s3.us-east-2.amazonaws.com/reports/pr-${PR_NUMBER}/${RUN_ID}/${PROJECT}/index.html"
	REPORT_LINK="[View Report](${REPORT_URL})"
	else
	REPORT_LINK="✅ No changes"
	fi

	TABLE_ROWS="${TABLE_ROWS}\| \`${PROJECT}\` \| ${CHANGED} \| ${ADDED} \| ${REMOVED} \| ${UNCHANGED} \| ${REPORT_LINK} \|\n"
	done

	if [ "${HAS_ANY_SUMMARY}" = "false" ]; then
	echo "No visual diff summaries found — skipping PR comment."
	exit 0
	fi

	BODY=$(printf '%s\n' \
	"${MARKER}" \
	"### 🖼️ Visual Regression Report" \
	"" \
	"${TABLE_HEADER}" \
	"${TABLE_DIVIDER}" \
	"$(printf '%b' "${TABLE_ROWS}")")

	# Upsert: find existing comment with the marker, or create a new one
	EXISTING_COMMENT_ID=$(gh api \
	"repos/${REPO}/issues/${PR_NUMBER}/comments" \
	--jq ".[] \| select(.body \| startswith(\"${MARKER}\")) \| .id" \
	2>/dev/null \| head -1)

	if [ -n "${EXISTING_COMMENT_ID}" ]; then
	gh api \
	--method PATCH \
	"repos/${REPO}/issues/comments/${EXISTING_COMMENT_ID}" \
	-f body="${BODY}"
	else
	gh api \
	--method POST \
	"repos/${REPO}/issues/${PR_NUMBER}/comments" \
	-f body="${BODY}"
	fi

	playwright-required:
	# NOTE: Github-hosted runners have about 20s faster queue times and are preferred here.
	runs-on: ubuntu-slim
	timeout-minutes: 45
	needs: [playwright-tests]
	if: ${{ always() }}
	steps:
	- name: Check job status
	if: ${{ contains(needs..result, 'failure') \|\| contains(needs..result, 'cancelled') \|\| contains(needs.*.result, 'skipped') }}
	run: exit 1

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Run Playwright Tests #20160

Workflow file

Run Playwright Tests #20160

Uh oh!

Workflow file for this run